From a5a5c9f476bb562afebf19cddeb002e0c2e6be3d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 4 Jun 2020 18:36:20 -0700 Subject: [PATCH 01/14] json custom serialization; application for strtabs --- lib/pure/json.nim | 15 +++++++++++++++ lib/pure/strtabs.nim | 13 +++++++++++++ tests/stdlib/tjsonmacro.nim | 9 +++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 58a8c1dcec233..a9072ff6dcb8f 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1287,3 +1287,18 @@ when false: # { "json": 5 } # To get that we shall use, obj["json"] + +proc toJson2*[T](a: T): JsonNode = + ## allows custom serialization + when compiles(serialize(a)): + var ret = newJObject() + proc dump(key: string, val: string) = + ret[key] = %val + serialize(a, dump) + result = ret + elif T is object: + result = newJObject() + for k,v in fieldPairs(a): + result[k] = toJson2(v) + else: + result = %a diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index cfafdf018efa3..cbda228f995c5 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -419,6 +419,19 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. add(result, f[i]) inc(i) +proc serialize*(a: StringTableRef) = discard +proc serialize*[Fun](a: StringTableRef, fun: Fun) = + ## allows a custom serializer (eg json) to serialize this as we want. + #[ + another option is to nest, but this is more compact. eg: + fun("mode", $a.mode) + fun("table") # open scope + for k,v in a: + fun(k, v) + fun() # close scope + ]# + fun("mode", $a.mode) + for k,v in a: fun(k, v) when isMainModule: var x = {"k": "v", "11": "22", "565": "67"}.newStringTable diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 938030d8ee8f1..09a21bb6e1956 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -633,8 +633,13 @@ proc testJson() = except KeyError: doAssert getCurrentExceptionMsg().contains ".member.list[2].value" - - testJson() static: testJson() + +import strtabs +proc testCustom()= + var t = {"name": "John", "city": "Monaco"}.newStringTable + let s = toJson2(t) + doAssert $s == """{"mode":"modeCaseSensitive","city":"Monaco","name":"John"}""" +testCustom() From 126ee2b9932888a64e3e48a3487c3c9c87a38d90 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 5 Jun 2020 00:30:16 -0700 Subject: [PATCH 02/14] serialize using nesting --- lib/pure/json.nim | 9 ++++----- lib/pure/strtabs.nim | 17 ++++++----------- tests/stdlib/tjsonmacro.nim | 2 +- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index a9072ff6dcb8f..82c1125f570e9 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1291,11 +1291,10 @@ when false: proc toJson2*[T](a: T): JsonNode = ## allows custom serialization when compiles(serialize(a)): - var ret = newJObject() - proc dump(key: string, val: string) = - ret[key] = %val - serialize(a, dump) - result = ret + proc funAdd(t: JsonNode, key: string, val: string) = t[key] = %val + proc funAdd2(t: JsonNode, key: string, val: JsonNode) = t[key] = val + proc funObj(): JsonNode = newJObject() + result = serialize(a, funObj, funAdd, funAdd2) elif T is object: result = newJObject() for k,v in fieldPairs(a): diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index cbda228f995c5..8ab3a1aa7b776 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -420,18 +420,13 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. inc(i) proc serialize*(a: StringTableRef) = discard -proc serialize*[Fun](a: StringTableRef, fun: Fun) = +proc serialize*[T1, T2, T3](a: StringTableRef, funObj: T1, funAdd: T2, funAdd2: T3): auto = ## allows a custom serializer (eg json) to serialize this as we want. - #[ - another option is to nest, but this is more compact. eg: - fun("mode", $a.mode) - fun("table") # open scope - for k,v in a: - fun(k, v) - fun() # close scope - ]# - fun("mode", $a.mode) - for k,v in a: fun(k, v) + result = funObj() + funAdd(result, "mode", $a.mode) + let t = funObj() + for k,v in a: funAdd(t, k, v) + funAdd2(result, "table", t) when isMainModule: var x = {"k": "v", "11": "22", "565": "67"}.newStringTable diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 09a21bb6e1956..6bb4be23e41aa 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -641,5 +641,5 @@ import strtabs proc testCustom()= var t = {"name": "John", "city": "Monaco"}.newStringTable let s = toJson2(t) - doAssert $s == """{"mode":"modeCaseSensitive","city":"Monaco","name":"John"}""" + doAssert $s == """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" testCustom() From 75bc412f732868fd8166bab67867ba6be1cf89fa Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 5 Jun 2020 02:06:02 -0700 Subject: [PATCH 03/14] make toJson more feature complete --- lib/pure/json.nim | 43 +++++++++++++++++++++++++++---------- tests/stdlib/tjsonmacro.nim | 13 ++++++++++- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 82c1125f570e9..c5390dd284c5d 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -378,20 +378,20 @@ proc `%`*(o: enum): JsonNode = ## string. Creates a new ``JString JsonNode``. result = %($o) -proc toJson(x: NimNode): NimNode {.compileTime.} = +proc toJsonImpl(x: NimNode): NimNode {.compileTime.} = case x.kind of nnkBracket: # array if x.len == 0: return newCall(bindSym"newJArray") result = newNimNode(nnkBracket) for i in 0 ..< x.len: - result.add(toJson(x[i])) + result.add(toJsonImpl(x[i])) result = newCall(bindSym("%", brOpen), result) of nnkTableConstr: # object if x.len == 0: return newCall(bindSym"newJObject") result = newNimNode(nnkTableConstr) for i in 0 ..< x.len: x[i].expectKind nnkExprColonExpr - result.add newTree(nnkExprColonExpr, x[i][0], toJson(x[i][1])) + result.add newTree(nnkExprColonExpr, x[i][0], toJsonImpl(x[i][1])) result = newCall(bindSym("%", brOpen), result) of nnkCurly: # empty object x.expectLen(0) @@ -399,7 +399,7 @@ proc toJson(x: NimNode): NimNode {.compileTime.} = of nnkNilLit: result = newCall(bindSym"newJNull") of nnkPar: - if x.len == 1: result = toJson(x[0]) + if x.len == 1: result = toJsonImpl(x[0]) else: result = newCall(bindSym("%", brOpen), x) else: result = newCall(bindSym("%", brOpen), x) @@ -407,7 +407,7 @@ proc toJson(x: NimNode): NimNode {.compileTime.} = macro `%*`*(x: untyped): untyped = ## Convert an expression to a JsonNode directly, without having to specify ## `%` for every element. - result = toJson(x) + result = toJsonImpl(x) proc `==`*(a, b: JsonNode): bool = ## Check two nodes for equality @@ -1288,16 +1288,37 @@ when false: # { "json": 5 } # To get that we shall use, obj["json"] -proc toJson2*[T](a: T): JsonNode = - ## allows custom serialization +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +proc toJson*[T](a: T): JsonNode = + ## like `%` but allows custom serialization hook if `serialize(a: T)` is in scope when compiles(serialize(a)): proc funAdd(t: JsonNode, key: string, val: string) = t[key] = %val proc funAdd2(t: JsonNode, key: string, val: JsonNode) = t[key] = val proc funObj(): JsonNode = newJObject() result = serialize(a, funObj, funAdd, funAdd2) - elif T is object: - result = newJObject() - for k,v in fieldPairs(a): - result[k] = toJson2(v) + elif T is object | tuple: + const isNamed = T is object or isNamedTuple(T) + when isNamed: + result = newJObject() + for k, v in a.fieldPairs: result[k] = toJson(v) + else: + result = newJArray() + for v in a.fields: result.add toJson(v) + elif T is ref | ptr: + if a == nil: result = newJNull() + else: result = toJson(a[]) + elif T is array | seq: + result = newJArray() + for ai in a: + result.add toJson(ai) + elif T is pointer: + result = toJson(cast[int](a)) + elif T is distinct: + result = toJson(a.distinctBase) + elif T is bool: + result = %(a) + elif T is Ordinal: + result = %(cast[int](a)) else: result = %a diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 6bb4be23e41aa..a5a1b53a23208 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -638,8 +638,19 @@ static: testJson() import strtabs + proc testCustom()= var t = {"name": "John", "city": "Monaco"}.newStringTable - let s = toJson2(t) + let s = toJson(t) doAssert $s == """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" + +proc testToJson() = + var t = {"z": "Z", "y": "Y"}.newStringTable + type A = ref object + a1: string + let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) + let j = a.toJson + doAssert $j == """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" + testCustom() +testToJson() From 4a5861f34e8ad5c30178f2dc60abc6eade96119f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 5 Jun 2020 02:39:48 -0700 Subject: [PATCH 04/14] add since --- lib/pure/json.nim | 3 ++- testament/important_packages.nim | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index c5390dd284c5d..9d6f04687e169 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -151,6 +151,7 @@ runnableExamples: import hashes, tables, strtabs, strutils, lexbase, streams, macros, parsejson, options +import std/private/since export tables.`$` @@ -1290,7 +1291,7 @@ when false: proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -proc toJson*[T](a: T): JsonNode = +proc toJson*[T](a: T): JsonNode {.since: (1,3,5).} = ## like `%` but allows custom serialization hook if `serialize(a: T)` is in scope when compiles(serialize(a)): proc funAdd(t: JsonNode, key: string, val: string) = t[key] = %val diff --git a/testament/important_packages.nim b/testament/important_packages.nim index cb436bfb2f81e..90e330f4c137b 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -53,7 +53,7 @@ pkg1 "illwill", false, "nimble examples" pkg1 "inim", true # pending https://github.com/inim-repl/INim/issues/74 pkg1 "itertools", false, "nim doc src/itertools.nim" pkg1 "iterutils" -pkg1 "jstin" +# pkg1 "jstin" # toJson clashes, jstin needs to be patched (eg: import except toJson) pkg1 "karax", false, "nim c -r tests/tester.nim" pkg1 "kdtree", false, "nimble test", "https://github.com/jblindsay/kdtree" pkg1 "loopfusion" From e36c23efcff085e363ab6e2cecbf3893fcddddd9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 5 Jun 2020 18:49:37 -0700 Subject: [PATCH 05/14] Revert "Improve JSON serialisation of strtabs (#14549)" This reverts commit 7cb4ef26addb3bb5ce2405d8396df6fd41664dae. --- lib/pure/json.nim | 35 ++++++----------------------------- lib/pure/strtabs.nim | 2 +- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 9d6f04687e169..b875af7dd4428 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -149,7 +149,7 @@ runnableExamples: doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" import - hashes, tables, strtabs, strutils, lexbase, streams, macros, parsejson, + hashes, tables, strutils, lexbase, streams, macros, parsejson, options import std/private/since @@ -354,14 +354,6 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = assert(obj.kind == JObject) obj.fields[key] = val -proc `%`*(table: StringTableRef): JsonNode = - ## Generic constructor for JSON data. Creates a new ``JObject JsonNode``. - result = newJObject() - result["mode"] = %($table.mode) - var data = newJObject() - for k, v in table: data[k] = %v - result["data"] = data - proc `%`*[T: object](o: T): JsonNode = ## Construct JsonNode from tuples and objects. result = newJObject() @@ -991,12 +983,11 @@ when defined(nimFixedForwardGeneric): proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var StringTableRef; jsonNode: JsonNode; jsonPath: var string) + proc initFromJson[T](dst: var Table[string,T];jsonNode: JsonNode; jsonPath: var string) + proc initFromJson[T](dst: var OrderedTable[string,T];jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) + proc initFromJson[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) # initFromJson definitions @@ -1054,7 +1045,7 @@ when defined(nimFixedForwardGeneric): initFromJson(dst[i], jsonNode[i], jsonPath) jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + proc initFromJson[T](dst: var Table[string,T];jsonNode: JsonNode; jsonPath: var string) = dst = initTable[string, T]() verifyJsonKind(jsonNode, {JObject}, jsonPath) let originalJsonPathLen = jsonPath.len @@ -1064,7 +1055,7 @@ when defined(nimFixedForwardGeneric): initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + proc initFromJson[T](dst: var OrderedTable[string,T];jsonNode: JsonNode; jsonPath: var string) = dst = initOrderedTable[string,T]() verifyJsonKind(jsonNode, {JObject}, jsonPath) let originalJsonPathLen = jsonPath.len @@ -1074,20 +1065,6 @@ when defined(nimFixedForwardGeneric): initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) jsonPath.setLen originalJsonPathLen - proc mgetOrPut(tab: var StringTableRef, key: string): var string = - if not tab.hasKey(key): tab[key] = "" - result = tab[key] - - proc initFromJson(dst: var StringTableRef; jsonNode: JsonNode; jsonPath: var string) = - dst = newStringTable(parseEnum[StringTableMode](jsonNode["mode"].getStr)) - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode["data"].fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) if jsonNode.kind == JNull: diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 8ab3a1aa7b776..7e814a585c33e 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -71,7 +71,7 @@ type StringTableObj* = object of RootObj counter: int data: KeyValuePairSeq - mode*: StringTableMode + mode: StringTableMode StringTableRef* = ref StringTableObj From 3fcf99344e5ebc510b2d2b4274dcf48637f70693 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 5 Jun 2020 19:13:32 -0700 Subject: [PATCH 06/14] better approach via mixin --- lib/pure/json.nim | 7 ++----- lib/pure/strtabs.nim | 16 +++++++++------- tests/stdlib/tjsonmacro.nim | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b875af7dd4428..70bda44e974e5 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1270,11 +1270,8 @@ proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} proc toJson*[T](a: T): JsonNode {.since: (1,3,5).} = ## like `%` but allows custom serialization hook if `serialize(a: T)` is in scope - when compiles(serialize(a)): - proc funAdd(t: JsonNode, key: string, val: string) = t[key] = %val - proc funAdd2(t: JsonNode, key: string, val: JsonNode) = t[key] = val - proc funObj(): JsonNode = newJObject() - result = serialize(a, funObj, funAdd, funAdd2) + when compiles(toJsonHook(a)): + result = toJsonHook(a) elif T is object | tuple: const isNamed = T is object or isNamedTuple(T) when isNamed: diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 7e814a585c33e..cc1df7b0cd12f 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -419,14 +419,16 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. add(result, f[i]) inc(i) -proc serialize*(a: StringTableRef) = discard -proc serialize*[T1, T2, T3](a: StringTableRef, funObj: T1, funAdd: T2, funAdd2: T3): auto = +proc toJsonHook*[](a: StringTableRef): auto = ## allows a custom serializer (eg json) to serialize this as we want. - result = funObj() - funAdd(result, "mode", $a.mode) - let t = funObj() - for k,v in a: funAdd(t, k, v) - funAdd2(result, "table", t) + mixin newJObject + mixin toJson + result = newJObject() + result["mode"] = toJson($a.mode) + let t = newJObject() + for k,v in a: + t[k] = toJson(v) + result["table"] = t when isMainModule: var x = {"k": "v", "11": "22", "565": "67"}.newStringTable diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index a5a1b53a23208..e2ac657843243 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -642,7 +642,7 @@ import strtabs proc testCustom()= var t = {"name": "John", "city": "Monaco"}.newStringTable let s = toJson(t) - doAssert $s == """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" + doAssert $s == """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""", $s proc testToJson() = var t = {"z": "Z", "y": "Y"}.newStringTable From 50d61c3623fd8f812caf766146c6cdbb1a653c70 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 5 Jun 2020 20:04:41 -0700 Subject: [PATCH 07/14] toJson, jsonTo --- changelog.md | 2 + lib/pure/json.nim | 104 ++++++++++++++++++++++++++++++------ lib/pure/strtabs.nim | 28 ++++++---- tests/stdlib/tjsonmacro.nim | 37 +++++++------ 4 files changed, 130 insertions(+), 41 deletions(-) diff --git a/changelog.md b/changelog.md index bbcf2ed183272..dc5dfd1e28be8 100644 --- a/changelog.md +++ b/changelog.md @@ -108,6 +108,8 @@ users from the use of weak and insecure ciphers while still provides adequate compatiblity with the majority of the Internet. +- added `json.jsonTo,toJson` for json serialization/deserialization from custom types. + ## Language changes - In the newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this: diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 70bda44e974e5..ce7e3096902a6 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -149,8 +149,9 @@ runnableExamples: doAssert $(%* Foo()) == """{"a1":0,"a2":0,"a0":0,"a3":0,"a4":0}""" import - hashes, tables, strutils, lexbase, streams, macros, parsejson, - options + hashes, tables, strutils, lexbase, streams, macros, parsejson + +import options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 import std/private/since export @@ -1267,11 +1268,88 @@ when false: # To get that we shall use, obj["json"] proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} +proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T): untyped = distinctBase(type(a))(a) + +proc checkJsonImpl(cond: bool, condStr: string, msg = "") = + if not cond: + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, msg) + +template checkJson(cond: untyped, msg = "") = + checkJsonImpl(cond, astToStr(cond), msg) + +proc fromJson*[T](a: var T, b: JsonNode) {.since: (1,3,5).} = + ## inplace version of `jsonTo` + #[ + adding "json path" leading to `b` can be added in future work. + ]# + checkJson b != nil, $($T, b) + when false: discard + elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + elif T is bool: a = to(b,T) + elif T is Table | OrderedTable: + a.clear + for k,v in b: + a[k] = jsonTo(v, typeof(a[k])) + elif T is enum: + case b.kind + of JInt: a = T(b.getBiggestInt()) + of JString: a = parseEnum[T](b.getStr()) + else: checkJson false, $($T, " ", b) + elif T is Ordinal: a = T(to(b, int)) + elif T is pointer: a = cast[pointer](to(b, int)) + elif T is distinct: a.distinctBase.fromJson(b) + elif T is string|SomeNumber: a = to(b,T) + elif T is JsonNode: a = b + elif T is ref | ptr: + if b.kind == JNull: a = nil + else: + a = T() + fromJson(a[], b) + elif T is array: + checkJson a.len == b.len, $(a.len, b.len, $T) + for i, val in b.getElems: + fromJson(a[i], val) + elif T is seq: + a.setLen b.len + for i, val in b.getElems: + fromJson(a[i], val) + elif T is object | tuple: + const isNamed = T is object or isNamedTuple(T) + when isNamed: + checkJson b.kind == JObject, $(b.kind) # we could customize whether to allow JNull + var num = 0 + for key, val in fieldPairs(a): + num.inc + if b.hasKey key: + fromJson(val, b[key]) + else: + # we could customize to allow this + checkJson false, $($T, key, b) + checkJson b.len == num, $(b.len, num, $T, b) # could customize + else: + checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + var i = 0 + for val in fields(a): + fromJson(val, b[i]) + i.inc + else: + # checkJson not appropriate here + static: doAssert false, "not yet implemented: " & $T + +proc jsonTo*(b: JsonNode, T: typedesc): T {.since: (1,3,5).} = + ## reverse of `toJson` + fromJson(result, b) proc toJson*[T](a: T): JsonNode {.since: (1,3,5).} = ## like `%` but allows custom serialization hook if `serialize(a: T)` is in scope - when compiles(toJsonHook(a)): - result = toJsonHook(a) + when false: discard + elif compiles(toJsonHook(a)): result = toJsonHook(a) + elif T is Table | OrderedTable: + result = newJObject() + for k, v in pairs(a): result[k] = toJson(v) elif T is object | tuple: const isNamed = T is object or isNamedTuple(T) when isNamed: @@ -1285,15 +1363,9 @@ proc toJson*[T](a: T): JsonNode {.since: (1,3,5).} = else: result = toJson(a[]) elif T is array | seq: result = newJArray() - for ai in a: - result.add toJson(ai) - elif T is pointer: - result = toJson(cast[int](a)) - elif T is distinct: - result = toJson(a.distinctBase) - elif T is bool: - result = %(a) - elif T is Ordinal: - result = %(cast[int](a)) - else: - result = %a + for ai in a: result.add toJson(ai) + elif T is pointer: result = toJson(cast[int](a)) + elif T is distinct: result = toJson(a.distinctBase) + elif T is bool: result = %(a) + elif T is Ordinal: result = %(cast[int](a)) + else: result = %a diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index cc1df7b0cd12f..81ff7fbdeb514 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -419,16 +419,24 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. add(result, f[i]) inc(i) -proc toJsonHook*[](a: StringTableRef): auto = - ## allows a custom serializer (eg json) to serialize this as we want. - mixin newJObject - mixin toJson - result = newJObject() - result["mode"] = toJson($a.mode) - let t = newJObject() - for k,v in a: - t[k] = toJson(v) - result["table"] = t +since (1,3,5): + proc fromJsonHook*[T](a: var StringTableRef, b: T) = + ## for json.fromJson + mixin jsonTo + var mode = jsonTo(b["mode"], StringTableMode) + a = newStringTable(mode) + let b2 = b["table"] + for k,v in b2: a[k] = jsonTo(v, string) + + proc toJsonHook*[](a: StringTableRef): auto = + ## for json.toJson + mixin newJObject + mixin toJson + result = newJObject() + result["mode"] = toJson($a.mode) + let t = newJObject() + for k,v in a: t[k] = toJson(v) + result["table"] = t when isMainModule: var x = {"k": "v", "11": "22", "565": "67"}.newStringTable diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index e2ac657843243..62ae46a5703d1 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -639,18 +639,25 @@ static: import strtabs -proc testCustom()= - var t = {"name": "John", "city": "Monaco"}.newStringTable - let s = toJson(t) - doAssert $s == """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""", $s - -proc testToJson() = - var t = {"z": "Z", "y": "Y"}.newStringTable - type A = ref object - a1: string - let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) - let j = a.toJson - doAssert $j == """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" - -testCustom() -testToJson() +# xxx move to `tjson` pending https://github.com/nim-lang/Nim/pull/14572 +proc testRoundtrip[T](t: T, expected: string) = + let j = t.toJson + doAssert $j == expected, $j + doAssert j.jsonTo(T).toJson == j + +block: # toJson, jsonTo + type Foo = distinct float + testRoundtrip('x', """120""") + when not defined(js): + testRoundtrip(cast[pointer](12345)): """12345""" + testRoundtrip(Foo(1.5)): """1.5""" + testRoundtrip({"z": "Z", "y": "Y"}.toOrderedTable): """{"z":"Z","y":"Y"}""" + testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}""" + testRoundtrip({"name": "John", "city": "Monaco"}.newStringTable): """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" + block: # complex example + let t = {"z": "Z", "y": "Y"}.newStringTable + type A = ref object + a1: string + let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'u8,3'u8], (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) + testRoundtrip(a): + """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" From ddbb3a764fef27f26c6480fcb4e7127d156cea08 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 5 Jun 2020 23:54:09 -0700 Subject: [PATCH 08/14] fix test --- lib/pure/json.nim | 2 +- tests/stdlib/tjsonmacro.nim | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index ce7e3096902a6..f690f33b68b85 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1367,5 +1367,5 @@ proc toJson*[T](a: T): JsonNode {.since: (1,3,5).} = elif T is pointer: result = toJson(cast[int](a)) elif T is distinct: result = toJson(a.distinctBase) elif T is bool: result = %(a) - elif T is Ordinal: result = %(cast[int](a)) + elif T is Ordinal: result = %(a.ord) else: result = %a diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 62ae46a5703d1..5cd0fe289431c 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -652,12 +652,13 @@ block: # toJson, jsonTo testRoundtrip(cast[pointer](12345)): """12345""" testRoundtrip(Foo(1.5)): """1.5""" testRoundtrip({"z": "Z", "y": "Y"}.toOrderedTable): """{"z":"Z","y":"Y"}""" - testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}""" + when not defined(js): # pending https://github.com/nim-lang/Nim/issues/14574 + testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}""" testRoundtrip({"name": "John", "city": "Monaco"}.newStringTable): """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" block: # complex example let t = {"z": "Z", "y": "Y"}.newStringTable type A = ref object a1: string - let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'u8,3'u8], (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) + let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'i8,3'i8], -4'i16, (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) testRoundtrip(a): - """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" + """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],-4,{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" From e6cf8e867304191d580168c70519c7ed45e138b3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 6 Jun 2020 04:30:43 -0700 Subject: [PATCH 09/14] address comments --- lib/pure/json.nim | 9 ++++----- tests/stdlib/tjsonmacro.nim | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index f690f33b68b85..5dad3fe467eef 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1286,8 +1286,7 @@ proc fromJson*[T](a: var T, b: JsonNode) {.since: (1,3,5).} = adding "json path" leading to `b` can be added in future work. ]# checkJson b != nil, $($T, b) - when false: discard - elif compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + when compiles(fromJsonHook(a, b)): fromJsonHook(a, b) elif T is bool: a = to(b,T) elif T is Table | OrderedTable: a.clear @@ -1344,9 +1343,9 @@ proc jsonTo*(b: JsonNode, T: typedesc): T {.since: (1,3,5).} = fromJson(result, b) proc toJson*[T](a: T): JsonNode {.since: (1,3,5).} = - ## like `%` but allows custom serialization hook if `serialize(a: T)` is in scope - when false: discard - elif compiles(toJsonHook(a)): result = toJsonHook(a) + ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to + ## customize serialization, see strtabs.toJsonHook for an example. + when compiles(toJsonHook(a)): result = toJsonHook(a) elif T is Table | OrderedTable: result = newJObject() for k, v in pairs(a): result[k] = toJson(v) diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 5cd0fe289431c..ca89351676867 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -639,7 +639,6 @@ static: import strtabs -# xxx move to `tjson` pending https://github.com/nim-lang/Nim/pull/14572 proc testRoundtrip[T](t: T, expected: string) = let j = t.toJson doAssert $j == expected, $j From 51a367d90179a45beb0dc9cf2069dadd0b381f79 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 6 Jun 2020 06:14:38 -0700 Subject: [PATCH 10/14] move to jsonutils --- changelog.md | 1 + lib/pure/json.nim | 105 +--------------------------- lib/std/jsonutils.nim | 113 +++++++++++++++++++++++++++++++ testament/important_packages.nim | 2 +- tests/stdlib/tjsonmacro.nim | 25 ------- tests/stdlib/tjsonutils.nim | 33 +++++++++ 6 files changed, 151 insertions(+), 128 deletions(-) create mode 100644 lib/std/jsonutils.nim create mode 100644 tests/stdlib/tjsonutils.nim diff --git a/changelog.md b/changelog.md index dc5dfd1e28be8..945ec0960d618 100644 --- a/changelog.md +++ b/changelog.md @@ -109,6 +109,7 @@ adequate compatiblity with the majority of the Internet. - added `json.jsonTo,toJson` for json serialization/deserialization from custom types. +- new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` for json serialization/deserialization of custom types. ## Language changes - In the newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this: diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 5dad3fe467eef..3134f7ebddc99 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -140,6 +140,9 @@ ## var j2 = %* {"name": "Isaac", "books": ["Robot Dreams"]} ## j2["details"] = %* {"age":35, "pi":3.1415} ## echo j2 +## +## See also: std/jsonutils for hookable json serialization/deserialization +## of arbitrary types. runnableExamples: ## Note: for JObject, key ordering is preserved, unlike in some languages, @@ -1266,105 +1269,3 @@ when false: # { "json": 5 } # To get that we shall use, obj["json"] - -proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} -template distinctBase[T](a: T): untyped = distinctBase(type(a))(a) - -proc checkJsonImpl(cond: bool, condStr: string, msg = "") = - if not cond: - # just pick 1 exception type for simplicity; other choices would be: - # JsonError, JsonParser, JsonKindError - raise newException(ValueError, msg) - -template checkJson(cond: untyped, msg = "") = - checkJsonImpl(cond, astToStr(cond), msg) - -proc fromJson*[T](a: var T, b: JsonNode) {.since: (1,3,5).} = - ## inplace version of `jsonTo` - #[ - adding "json path" leading to `b` can be added in future work. - ]# - checkJson b != nil, $($T, b) - when compiles(fromJsonHook(a, b)): fromJsonHook(a, b) - elif T is bool: a = to(b,T) - elif T is Table | OrderedTable: - a.clear - for k,v in b: - a[k] = jsonTo(v, typeof(a[k])) - elif T is enum: - case b.kind - of JInt: a = T(b.getBiggestInt()) - of JString: a = parseEnum[T](b.getStr()) - else: checkJson false, $($T, " ", b) - elif T is Ordinal: a = T(to(b, int)) - elif T is pointer: a = cast[pointer](to(b, int)) - elif T is distinct: a.distinctBase.fromJson(b) - elif T is string|SomeNumber: a = to(b,T) - elif T is JsonNode: a = b - elif T is ref | ptr: - if b.kind == JNull: a = nil - else: - a = T() - fromJson(a[], b) - elif T is array: - checkJson a.len == b.len, $(a.len, b.len, $T) - for i, val in b.getElems: - fromJson(a[i], val) - elif T is seq: - a.setLen b.len - for i, val in b.getElems: - fromJson(a[i], val) - elif T is object | tuple: - const isNamed = T is object or isNamedTuple(T) - when isNamed: - checkJson b.kind == JObject, $(b.kind) # we could customize whether to allow JNull - var num = 0 - for key, val in fieldPairs(a): - num.inc - if b.hasKey key: - fromJson(val, b[key]) - else: - # we could customize to allow this - checkJson false, $($T, key, b) - checkJson b.len == num, $(b.len, num, $T, b) # could customize - else: - checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull - var i = 0 - for val in fields(a): - fromJson(val, b[i]) - i.inc - else: - # checkJson not appropriate here - static: doAssert false, "not yet implemented: " & $T - -proc jsonTo*(b: JsonNode, T: typedesc): T {.since: (1,3,5).} = - ## reverse of `toJson` - fromJson(result, b) - -proc toJson*[T](a: T): JsonNode {.since: (1,3,5).} = - ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to - ## customize serialization, see strtabs.toJsonHook for an example. - when compiles(toJsonHook(a)): result = toJsonHook(a) - elif T is Table | OrderedTable: - result = newJObject() - for k, v in pairs(a): result[k] = toJson(v) - elif T is object | tuple: - const isNamed = T is object or isNamedTuple(T) - when isNamed: - result = newJObject() - for k, v in a.fieldPairs: result[k] = toJson(v) - else: - result = newJArray() - for v in a.fields: result.add toJson(v) - elif T is ref | ptr: - if a == nil: result = newJNull() - else: result = toJson(a[]) - elif T is array | seq: - result = newJArray() - for ai in a: result.add toJson(ai) - elif T is pointer: result = toJson(cast[int](a)) - elif T is distinct: result = toJson(a.distinctBase) - elif T is bool: result = %(a) - elif T is Ordinal: result = %(a.ord) - else: result = %a diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim new file mode 100644 index 0000000000000..466f7c1e497f4 --- /dev/null +++ b/lib/std/jsonutils.nim @@ -0,0 +1,113 @@ +##[ +This module implements a hookable (de)serialization for arbitrary types. +]## + +import std/[json,tables,strutils] + +#[ +xxx +use toJsonHook,fromJsonHook for Table|OrderedTable +add Options support also using toJsonHook,fromJsonHook and remove `json=>options` dependency +]# + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} +proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T): untyped = distinctBase(type(a))(a) + +proc checkJsonImpl(cond: bool, condStr: string, msg = "") = + if not cond: + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, msg) + +template checkJson(cond: untyped, msg = "") = + checkJsonImpl(cond, astToStr(cond), msg) + +proc fromJson*[T](a: var T, b: JsonNode) = + ## inplace version of `jsonTo` + #[ + adding "json path" leading to `b` can be added in future work. + ]# + checkJson b != nil, $($T, b) + when compiles(fromJsonHook(a, b)): fromJsonHook(a, b) + elif T is bool: a = to(b,T) + elif T is Table | OrderedTable: + a.clear + for k,v in b: + a[k] = jsonTo(v, typeof(a[k])) + elif T is enum: + case b.kind + of JInt: a = T(b.getBiggestInt()) + of JString: a = parseEnum[T](b.getStr()) + else: checkJson false, $($T, " ", b) + elif T is Ordinal: a = T(to(b, int)) + elif T is pointer: a = cast[pointer](to(b, int)) + elif T is distinct: a.distinctBase.fromJson(b) + elif T is string|SomeNumber: a = to(b,T) + elif T is JsonNode: a = b + elif T is ref | ptr: + if b.kind == JNull: a = nil + else: + a = T() + fromJson(a[], b) + elif T is array: + checkJson a.len == b.len, $(a.len, b.len, $T) + for i, val in b.getElems: + fromJson(a[i], val) + elif T is seq: + a.setLen b.len + for i, val in b.getElems: + fromJson(a[i], val) + elif T is object | tuple: + const isNamed = T is object or isNamedTuple(T) + when isNamed: + checkJson b.kind == JObject, $(b.kind) # we could customize whether to allow JNull + var num = 0 + for key, val in fieldPairs(a): + num.inc + if b.hasKey key: + fromJson(val, b[key]) + else: + # we could customize to allow this + checkJson false, $($T, key, b) + checkJson b.len == num, $(b.len, num, $T, b) # could customize + else: + checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull + var i = 0 + for val in fields(a): + fromJson(val, b[i]) + i.inc + else: + # checkJson not appropriate here + static: doAssert false, "not yet implemented: " & $T + +proc jsonTo*(b: JsonNode, T: typedesc): T = + ## reverse of `toJson` + fromJson(result, b) + +proc toJson*[T](a: T): JsonNode = + ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to + ## customize serialization, see strtabs.toJsonHook for an example. + when compiles(toJsonHook(a)): result = toJsonHook(a) + elif T is Table | OrderedTable: + result = newJObject() + for k, v in pairs(a): result[k] = toJson(v) + elif T is object | tuple: + const isNamed = T is object or isNamedTuple(T) + when isNamed: + result = newJObject() + for k, v in a.fieldPairs: result[k] = toJson(v) + else: + result = newJArray() + for v in a.fields: result.add toJson(v) + elif T is ref | ptr: + if a == nil: result = newJNull() + else: result = toJson(a[]) + elif T is array | seq: + result = newJArray() + for ai in a: result.add toJson(ai) + elif T is pointer: result = toJson(cast[int](a)) + elif T is distinct: result = toJson(a.distinctBase) + elif T is bool: result = %(a) + elif T is Ordinal: result = %(a.ord) + else: result = %a diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 90e330f4c137b..cb436bfb2f81e 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -53,7 +53,7 @@ pkg1 "illwill", false, "nimble examples" pkg1 "inim", true # pending https://github.com/inim-repl/INim/issues/74 pkg1 "itertools", false, "nim doc src/itertools.nim" pkg1 "iterutils" -# pkg1 "jstin" # toJson clashes, jstin needs to be patched (eg: import except toJson) +pkg1 "jstin" pkg1 "karax", false, "nim c -r tests/tester.nim" pkg1 "kdtree", false, "nimble test", "https://github.com/jblindsay/kdtree" pkg1 "loopfusion" diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index ca89351676867..e7b0bdc261d52 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -636,28 +636,3 @@ proc testJson() = testJson() static: testJson() - -import strtabs - -proc testRoundtrip[T](t: T, expected: string) = - let j = t.toJson - doAssert $j == expected, $j - doAssert j.jsonTo(T).toJson == j - -block: # toJson, jsonTo - type Foo = distinct float - testRoundtrip('x', """120""") - when not defined(js): - testRoundtrip(cast[pointer](12345)): """12345""" - testRoundtrip(Foo(1.5)): """1.5""" - testRoundtrip({"z": "Z", "y": "Y"}.toOrderedTable): """{"z":"Z","y":"Y"}""" - when not defined(js): # pending https://github.com/nim-lang/Nim/issues/14574 - testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}""" - testRoundtrip({"name": "John", "city": "Monaco"}.newStringTable): """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" - block: # complex example - let t = {"z": "Z", "y": "Y"}.newStringTable - type A = ref object - a1: string - let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'i8,3'i8], -4'i16, (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) - testRoundtrip(a): - """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],-4,{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim new file mode 100644 index 0000000000000..9ed19541062dd --- /dev/null +++ b/tests/stdlib/tjsonutils.nim @@ -0,0 +1,33 @@ +import std/jsonutils +import std/json + +proc testRoundtrip[T](t: T, expected: string) = + let j = t.toJson + doAssert $j == expected, $j + doAssert j.jsonTo(T).toJson == j + +block: # toJson, jsonTo + type Foo = distinct float + testRoundtrip('x', """120""") + when not defined(js): + testRoundtrip(cast[pointer](12345)): """12345""" + testRoundtrip(Foo(1.5)): """1.5""" + + +import tables +block: + testRoundtrip({"z": "Z", "y": "Y"}.toOrderedTable): """{"z":"Z","y":"Y"}""" + when not defined(js): # pending https://github.com/nim-lang/Nim/issues/14574 + testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}""" + +import strtabs + +block: + testRoundtrip({"name": "John", "city": "Monaco"}.newStringTable): """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" + block: # complex example + let t = {"z": "Z", "y": "Y"}.newStringTable + type A = ref object + a1: string + let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'i8,3'i8], -4'i16, (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) + testRoundtrip(a): + """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],-4,{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" From af4e55df61abca6d367e0cc13d0ef2f262bdf859 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 6 Jun 2020 06:20:00 -0700 Subject: [PATCH 11/14] doc --- lib/std/jsonutils.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 466f7c1e497f4..6b28978fb5157 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -1,5 +1,8 @@ ##[ This module implements a hookable (de)serialization for arbitrary types. +Design goal: avoid importing modules where a custom serialization is needed; +see strtabs.fromJsonHook,toJsonHook for an example. + ]## import std/[json,tables,strutils] @@ -8,6 +11,11 @@ import std/[json,tables,strutils] xxx use toJsonHook,fromJsonHook for Table|OrderedTable add Options support also using toJsonHook,fromJsonHook and remove `json=>options` dependency + +future direction: +add a way to customize serialization, for eg allowing missing +or extra fields in JsonNode, field renaming, and a way to handle cyclic references +using a cache of already visited addresses. ]# proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} From 9a520038b3398412bae8d7a52dc524d7761f4af4 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 6 Jun 2020 18:24:46 -0700 Subject: [PATCH 12/14] cleanups --- changelog.md | 1 - lib/pure/json.nim | 10 +++++----- tests/stdlib/tjsonmacro.nim | 2 ++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index 945ec0960d618..8e06d30f196e9 100644 --- a/changelog.md +++ b/changelog.md @@ -108,7 +108,6 @@ users from the use of weak and insecure ciphers while still provides adequate compatiblity with the majority of the Internet. -- added `json.jsonTo,toJson` for json serialization/deserialization from custom types. - new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` for json serialization/deserialization of custom types. ## Language changes diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 3134f7ebddc99..041816c7ddc11 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -987,11 +987,11 @@ when defined(nimFixedForwardGeneric): proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string,T];jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string,T];jsonNode: JsonNode; jsonPath: var string) + proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) + proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) + proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) # initFromJson definitions @@ -1049,7 +1049,7 @@ when defined(nimFixedForwardGeneric): initFromJson(dst[i], jsonNode[i], jsonPath) jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var Table[string,T];jsonNode: JsonNode; jsonPath: var string) = + proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = dst = initTable[string, T]() verifyJsonKind(jsonNode, {JObject}, jsonPath) let originalJsonPathLen = jsonPath.len @@ -1059,7 +1059,7 @@ when defined(nimFixedForwardGeneric): initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) jsonPath.setLen originalJsonPathLen - proc initFromJson[T](dst: var OrderedTable[string,T];jsonNode: JsonNode; jsonPath: var string) = + proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = dst = initOrderedTable[string,T]() verifyJsonKind(jsonNode, {JObject}, jsonPath) let originalJsonPathLen = jsonPath.len diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index e7b0bdc261d52..938030d8ee8f1 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -633,6 +633,8 @@ proc testJson() = except KeyError: doAssert getCurrentExceptionMsg().contains ".member.list[2].value" + + testJson() static: testJson() From 6ec9a58004042a91974272bfdd5449bcd92702d3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 6 Jun 2020 18:27:12 -0700 Subject: [PATCH 13/14] also test for js --- tests/stdlib/tjsonutils.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index 9ed19541062dd..3bba84ab5a0b9 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -1,3 +1,7 @@ +discard """ + targets: "c cpp js" +""" + import std/jsonutils import std/json From 5a6f5941ece5d2aa842d44c402d72dbed799a232 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 6 Jun 2020 18:27:50 -0700 Subject: [PATCH 14/14] also test for vm --- lib/std/jsonutils.nim | 7 ++++++- tests/stdlib/tjsonutils.nim | 35 +++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 6b28978fb5157..bfa600fa91d4e 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -50,7 +50,12 @@ proc fromJson*[T](a: var T, b: JsonNode) = else: checkJson false, $($T, " ", b) elif T is Ordinal: a = T(to(b, int)) elif T is pointer: a = cast[pointer](to(b, int)) - elif T is distinct: a.distinctBase.fromJson(b) + elif T is distinct: + when nimvm: + # bug, potentially related to https://github.com/nim-lang/Nim/issues/12282 + a = T(jsonTo(b, distinctBase(T))) + else: + a.distinctBase.fromJson(b) elif T is string|SomeNumber: a = to(b,T) elif T is JsonNode: a = b elif T is ref | ptr: diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index 3bba84ab5a0b9..01c6aa05a0e5c 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -10,24 +10,28 @@ proc testRoundtrip[T](t: T, expected: string) = doAssert $j == expected, $j doAssert j.jsonTo(T).toJson == j -block: # toJson, jsonTo - type Foo = distinct float - testRoundtrip('x', """120""") - when not defined(js): - testRoundtrip(cast[pointer](12345)): """12345""" - testRoundtrip(Foo(1.5)): """1.5""" +import tables +import strtabs +template fn() = + block: # toJson, jsonTo + type Foo = distinct float + testRoundtrip('x', """120""") + when not defined(js): + testRoundtrip(cast[pointer](12345)): """12345""" -import tables -block: - testRoundtrip({"z": "Z", "y": "Y"}.toOrderedTable): """{"z":"Z","y":"Y"}""" - when not defined(js): # pending https://github.com/nim-lang/Nim/issues/14574 - testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}""" + # causes workaround in `fromJson` potentially related to + # https://github.com/nim-lang/Nim/issues/12282 + testRoundtrip(Foo(1.5)): """1.5""" -import strtabs + block: + testRoundtrip({"z": "Z", "y": "Y"}.toOrderedTable): """{"z":"Z","y":"Y"}""" + when not defined(js): # pending https://github.com/nim-lang/Nim/issues/14574 + testRoundtrip({"z": (f1: 'f'), }.toTable): """{"z":{"f1":102}}""" + + block: + testRoundtrip({"name": "John", "city": "Monaco"}.newStringTable): """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" -block: - testRoundtrip({"name": "John", "city": "Monaco"}.newStringTable): """{"mode":"modeCaseSensitive","table":{"city":"Monaco","name":"John"}}""" block: # complex example let t = {"z": "Z", "y": "Y"}.newStringTable type A = ref object @@ -35,3 +39,6 @@ block: let a = (1.1, "fo", 'x', @[10,11], [true, false], [t,newStringTable()], [0'i8,3'i8], -4'i16, (foo: 0.5'f32, bar: A(a1: "abc"), bar2: A.default)) testRoundtrip(a): """[1.1,"fo",120,[10,11],[true,false],[{"mode":"modeCaseSensitive","table":{"y":"Y","z":"Z"}},{"mode":"modeCaseSensitive","table":{}}],[0,3],-4,{"foo":0.5,"bar":{"a1":"abc"},"bar2":null}]""" + +static: fn() +fn()