alternative to fix #15413 using a trailing \0 encoding of unparseable numbers

changelog.md

@@ -8,6 +8,7 @@
 
 - Added `ioutils` module containing `duplicate` and `duplicateTo` to duplicate `FileHandle` using C function `dup` and `dup2`.
 
+- `json` now supports parsing numbers beyond `BiggestInt` range, via a specially encoded `JString`.
 
 ## Language changes
 

lib/pure/json.nim

@@ -165,6 +165,8 @@ export
   open, close, str, getInt, getFloat, kind, getColumn, getLine, getFilename,
   errorMsg, errorMsgExpected, next, JsonParsingError, raiseParseErr, nimIdentNormalize
 
+const numTerm = '\0'
+
 type
   JsonNodeKind* = enum ## possible JSON node types
     JNull,
@@ -201,6 +203,10 @@ proc newJStringMove(s: string): JsonNode =
   result = JsonNode(kind: JString)
   shallowCopy(result.str, s)
 
+proc newJLargeNumber*(n: string): JsonNode =
+  ## Creates a new `JString JsonNode` representing an un-parseable number.
+  result = JsonNode(kind: JString, str: n & numTerm)
+
 proc newJInt*(n: BiggestInt): JsonNode =
   ## Creates a new `JInt JsonNode`.
   result = JsonNode(kind: JInt, num: n)
@@ -225,13 +231,37 @@ proc newJArray*(): JsonNode =
   ## Creates a new `JArray JsonNode`
   result = JsonNode(kind: JArray, elems: @[])
 
+proc isLargeNumber*(n: JsonNode): bool =
+  n != nil and n.kind == JString and n.str.endsWith(numTerm)
+
+proc getLargeNumberUnchecked*(n: JsonNode): string =
+  n.str[0..^2]
+
+proc isBiggestUint*(n: JsonNode): bool =
+  if n.isLargeNumber:
+    try:
+      discard n.getLargeNumberUnchecked.parseBiggestUInt()
+      return true
+    except ValueError: return false
+  elif n!=nil and n.kind == JInt and n.num >= 0:
+    return true
+
 proc getStr*(n: JsonNode, default: string = ""): string =
   ## Retrieves the string value of a `JString JsonNode`.
   ##
   ## Returns ``default`` if ``n`` is not a ``JString``, or if ``n`` is nil.
   if n.isNil or n.kind != JString: return default
   else: return n.str
 
+proc getLargeNumber*(n: JsonNode, default: string = ""): string =
+  if n.isLargeNumber: n.getLargeNumberUnchecked
+  else: default
+
+proc getBiggestUInt*(n: JsonNode, default: BiggestUInt = 0): BiggestUInt =
+  if n.isLargeNumber: n.getLargeNumberUnchecked.parseBiggestUInt()
+  elif n!=nil and n.kind == JInt: n.num.BiggestUInt
+  else: default
+
 proc getInt*(n: JsonNode, default: int = 0): int =
   ## Retrieves the int value of a `JInt JsonNode`.
   ##
@@ -295,15 +325,21 @@ proc `%`*(s: string): JsonNode =
 
 proc `%`*(n: uint): JsonNode =
   ## Generic constructor for JSON data. Creates a new `JInt JsonNode`.
-  result = JsonNode(kind: JInt, num: BiggestInt(n))
+  if cast[int](n) < 0:
+    result = newJLargeNumber($n)
+  else:
+    result = JsonNode(kind: JInt, num: BiggestInt(n))
 
 proc `%`*(n: int): JsonNode =
   ## Generic constructor for JSON data. Creates a new `JInt JsonNode`.
   result = JsonNode(kind: JInt, num: n)
 
 proc `%`*(n: BiggestUInt): JsonNode =
   ## Generic constructor for JSON data. Creates a new `JInt JsonNode`.
-  result = JsonNode(kind: JInt, num: BiggestInt(n))
+  if cast[BiggestInt](n) < 0:
+    result = newJLargeNumber($n)
+  else:
+    result = JsonNode(kind: JInt, num: BiggestInt(n))
 
 proc `%`*(n: BiggestInt): JsonNode =
   ## Generic constructor for JSON data. Creates a new `JInt JsonNode`.
@@ -652,7 +688,10 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
       result.add("{}")
   of JString:
     if lstArr: result.indent(currIndent)
-    escapeJson(node.str, result)
+    if node.isLargeNumber:
+      result.add node.getLargeNumberUnchecked
+    else:
+      escapeJson(node.str, result)
   of JInt:
     if lstArr: result.indent(currIndent)
     when defined(js): result.add($node.num)
@@ -734,7 +773,10 @@ proc toUgly*(result: var string, node: JsonNode) =
       result.toUgly value
     result.add "}"
   of JString:
-    node.str.escapeJson(result)
+    if node.isLargeNumber:
+      result.add node.getLargeNumberUnchecked
+    else:
+      node.str.escapeJson(result)
   of JInt:
     when defined(js): result.add($node.num)
     else: result.addInt(node.num)
@@ -792,7 +834,10 @@ proc parseJson(p: var JsonParser): JsonNode =
     p.a = ""
     discard getTok(p)
   of tkInt:
-    result = newJInt(parseBiggestInt(p.a))
+    try:
+      result = newJInt(parseBiggestInt(p.a))
+    except ValueError:
+      result = newJLargeNumber(p.a)
     discard getTok(p)
   of tkFloat:
     result = newJFloat(parseFloat(p.a))
@@ -868,11 +913,9 @@ when defined(js):
 
   proc parseNativeJson(x: cstring): JSObject {.importc: "JSON.parse".}
 
-  proc getVarType(x: JSObject): JsonNodeKind =
+  proc getVarType2(proto: cstring, x: JSObject): JsonNodeKind =
     result = JNull
-    proc getProtoName(y: JSObject): cstring
-      {.importc: "Object.prototype.toString.call".}
-    case $getProtoName(x) # TODO: Implicit returns fail here.
+    case $proto # TODO: Implicit returns fail here.
     of "[object Array]": return JArray
     of "[object Object]": return JObject
     of "[object Number]":
@@ -883,8 +926,15 @@ when defined(js):
     of "[object Boolean]": return JBool
     of "[object Null]": return JNull
     of "[object String]": return JString
+    of "[object BigInt]": return JString
     else: assert false
 
+  proc getProtoName(y: JSObject): cstring
+    {.importc: "Object.prototype.toString.call".}
+
+  proc getVarType(x: JSObject): JsonNodeKind =
+    result = getVarType2(getProtoName(x), x)
+
   proc len(x: JSObject): int =
     assert x.getVarType == JArray
     asm """
@@ -904,31 +954,35 @@ when defined(js):
     """
 
   proc convertObject(x: JSObject): JsonNode =
-    case getVarType(x)
-    of JArray:
-      result = newJArray()
-      for i in 0 ..< x.len:
-        result.add(x[i].convertObject())
-    of JObject:
-      result = newJObject()
-      asm """for (var property in `x`) {
-        if (`x`.hasOwnProperty(property)) {
-      """
-      var nimProperty: cstring
-      var nimValue: JSObject
-      asm "`nimProperty` = property; `nimValue` = `x`[property];"
-      result[$nimProperty] = nimValue.convertObject()
-      asm "}}"
-    of JInt:
-      result = newJInt(cast[int](x))
-    of JFloat:
-      result = newJFloat(cast[float](x))
-    of JString:
-      result = newJString($cast[cstring](x))
-    of JBool:
-      result = newJBool(cast[bool](x))
-    of JNull:
-      result = newJNull()
+    let proto = getProtoName(x)
+    if proto == "[object BigInt]":
+       result = newJLargeNumber($x)
+    else:
+      case getVarType2(proto, x)
+      of JArray:
+        result = newJArray()
+        for i in 0 ..< x.len:
+          result.add(x[i].convertObject())
+      of JObject:
+        result = newJObject()
+        asm """for (var property in `x`) {
+          if (`x`.hasOwnProperty(property)) {
+        """
+        var nimProperty: cstring
+        var nimValue: JSObject
+        asm "`nimProperty` = property; `nimValue` = `x`[property];"
+        result[$nimProperty] = nimValue.convertObject()
+        asm "}}"
+      of JInt:
+        result = newJInt(cast[int](x))
+      of JFloat:
+        result = newJFloat(cast[float](x))
+      of JString:
+        result = newJString($cast[cstring](x))
+      of JBool:
+        result = newJBool(cast[bool](x))
+      of JNull:
+        result = newJNull()
 
   proc parseJson*(buffer: string): JsonNode =
     when nimvm:
@@ -1014,8 +1068,14 @@ when defined(nimFixedForwardGeneric):
     dst = jsonNode.copy
 
   proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) =
-    verifyJsonKind(jsonNode, {JInt}, jsonPath)
-    dst = T(jsonNode.num)
+    template fn() = 
+      verifyJsonKind(jsonNode, {JInt}, jsonPath)
+      dst = T(jsonNode.num)
+    when T is BiggestUInt:
+      if jsonNode.isBiggestUint:
+        dst = T(jsonNode.getBiggestUInt)
+      else: fn()
+    else: fn()
 
   proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
     verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath)

tests/stdlib/tjson.nim

@@ -3,6 +3,7 @@ Note: Macro tests are in tests/stdlib/tjsonmacro.nim
 ]#
 
 import std/[json,parsejson,strutils,streams]
+# when not defined 
 
 let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}"""
 # nil passthrough
@@ -198,13 +199,12 @@ block:
 
   doAssert(obj == to(%obj, type(obj)))
 
-  when not defined(js):
-    const fragments = """[1,2,3] {"hi":3} 12 [] """
-    var res = ""
-    for x in parseJsonFragments(newStringStream(fragments)):
-      res.add($x)
-      res.add " "
-    doAssert res == fragments
+  const fragments = """[1,2,3] {"hi":3} 12 [] """
+  var res = ""
+  for x in parseJsonFragments(newStringStream(fragments)):
+    res.add($x)
+    res.add " "
+  doAssert res == fragments
 
 
 # test isRefSkipDistinct
@@ -232,3 +232,29 @@ doAssert isRefSkipDistinct(MyRef)
 doAssert not isRefSkipDistinct(MyObject)
 doAssert isRefSkipDistinct(MyDistinct)
 doAssert isRefSkipDistinct(MyOtherDistinct)
+
+template main() = 
+  # xxx put everything inside `main` so it can be tested with and without static
+  block: # uint64; bug #15413
+    when not defined(js):
+      let a = 18446744073709551605'u64
+      doAssert a > cast[uint64](int64.high)
+      let s = $a
+      let j = parseJson(s)
+      doAssert j.isBiggestUInt
+      doAssert $j == s
+      doAssert j.pretty == s
+      doAssert j.getBiggestUInt == a
+
+  block: # BigInt
+    let s = "184467440737095516151"
+    let j = parseJson(s)
+    when not defined(js):
+      doAssert uint64.high == 18446744073709551615'u64
+      doAssert j.isLargeNumber
+      doAssert $j == s
+      doAssert j.pretty == s
+      doAssert j.getLargeNumber == s
+
+main()
+static: main()