Skip to content

Commit

Permalink
fixes #15413
Browse files Browse the repository at this point in the history
  • Loading branch information
@Araq
Araq committed Oct 29, 2020
1 parent 485d4ff commit c1f58d3
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 22 deletions.
6 changes: 6 additions & 0 deletions changelog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@

- Added `ioutils` module containing `duplicate` and `duplicateTo` to duplicate `FileHandle` using C function `dup` and `dup2`.

- The JSON module can now handle integer literals and floating point literals of arbitrary length and precision.
Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are kept as string literals and
one can use external BigNum libraries to handle these. The `parseFloat` family of functions also has now optional
`rawIntegers` and `rawFloats` parameters that can be used to enforce that all integer or float literals remain
in the "raw" string form so that client code can easily treat small and large numbers uniformly.


## Language changes

Expand Down
88 changes: 66 additions & 22 deletions lib/pure/json.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,8 @@ type

JsonNode* = ref JsonNodeObj ## JSON node
JsonNodeObj* {.acyclic.} = object
isUnquoted: bool # the JString was a number-like token and
# so shouldn't be quoted
case kind*: JsonNodeKind
of JString:
str*: string
Expand All @@ -197,6 +199,13 @@ proc newJString*(s: string): JsonNode =
## Creates a new `JString JsonNode`.
result = JsonNode(kind: JString, str: s)

proc newJRawNumber*(s: string): JsonNode {.since: (1, 5).} =
## Creates a "raw JS number", that is a number that does not
## fit into Nim's ``BiggestInt`` field. This is really a `JString`
## with the additional information that it should be converted back
## to the string representation without the quotes.
result = JsonNode(kind: JString, str: s, isUnquoted: true)

proc newJStringMove(s: string): JsonNode =
result = JsonNode(kind: JString)
shallowCopy(result.str, s)
Expand Down Expand Up @@ -562,6 +571,7 @@ proc copy*(p: JsonNode): JsonNode =
case p.kind
of JString:
result = newJString(p.str)
result.isUnquoted = p.isUnquoted
of JInt:
result = newJInt(p.num)
of JFloat:
Expand Down Expand Up @@ -652,7 +662,10 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
result.add("{}")
of JString:
if lstArr: result.indent(currIndent)
escapeJson(node.str, result)
if node.isUnquoted:
result.add node.str
else:
escapeJson(node.str, result)
of JInt:
if lstArr: result.indent(currIndent)
when defined(js): result.add($node.num)
Expand Down Expand Up @@ -734,7 +747,10 @@ proc toUgly*(result: var string, node: JsonNode) =
result.toUgly value
result.add "}"
of JString:
node.str.escapeJson(result)
if node.isUnquoted:
result.add node.str
else:
node.str.escapeJson(result)
of JInt:
when defined(js): result.add($node.num)
else: result.addInt(node.num)
Expand Down Expand Up @@ -783,7 +799,7 @@ iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] =
for key, val in mpairs(node.fields):
yield (key, val)

proc parseJson(p: var JsonParser): JsonNode =
proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode =
## Parses JSON from a JSON Parser `p`.
case p.tok
of tkString:
Expand All @@ -792,10 +808,22 @@ proc parseJson(p: var JsonParser): JsonNode =
p.a = ""
discard getTok(p)
of tkInt:
result = newJInt(parseBiggestInt(p.a))
if rawIntegers:
result = newJRawNumber(p.a)
else:
try:
result = newJInt(parseBiggestInt(p.a))
except ValueError:
result = newJRawNumber(p.a)
discard getTok(p)
of tkFloat:
result = newJFloat(parseFloat(p.a))
if rawFloats:
result = newJRawNumber(p.a)
else:
try:
result = newJFloat(parseFloat(p.a))
except ValueError:
result = newJRawNumber(p.a)
discard getTok(p)
of tkTrue:
result = newJBool(true)
Expand All @@ -815,7 +843,7 @@ proc parseJson(p: var JsonParser): JsonNode =
var key = p.a
discard getTok(p)
eat(p, tkColon)
var val = parseJson(p)
var val = parseJson(p, rawIntegers, rawFloats)
result[key] = val
if p.tok != tkComma: break
discard getTok(p)
Expand All @@ -824,39 +852,47 @@ proc parseJson(p: var JsonParser): JsonNode =
result = newJArray()
discard getTok(p)
while p.tok != tkBracketRi:
result.add(parseJson(p))
result.add(parseJson(p, rawIntegers, rawFloats))
if p.tok != tkComma: break
discard getTok(p)
eat(p, tkBracketRi)
of tkError, tkCurlyRi, tkBracketRi, tkColon, tkComma, tkEof:
raiseParseErr(p, "{")

iterator parseJsonFragments*(s: Stream, filename: string = ""): JsonNode =
iterator parseJsonFragments*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats = false): JsonNode =
## Parses from a stream `s` into `JsonNodes`. `filename` is only needed
## for nice error messages.
## The JSON fragments are separated by whitespace. This can be substantially
## faster than the comparable loop
## ``for x in splitWhitespace(s): yield parseJson(x)``.
## This closes the stream `s` after it's done.
## If `rawIntegers` is true, integer literals will not be converted to a `JInt`
## field but kept as raw numbers via `JString`.
## If `rawFloats` is true, floating point literals will not be converted to a `JFloat`
## field but kept as raw numbers via `JString`.
var p: JsonParser
p.open(s, filename)
try:
discard getTok(p) # read first token
while p.tok != tkEof:
yield p.parseJson()
yield p.parseJson(rawIntegers, rawFloats)
finally:
p.close()

proc parseJson*(s: Stream, filename: string = ""): JsonNode =
proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats = false): JsonNode =
## Parses from a stream `s` into a `JsonNode`. `filename` is only needed
## for nice error messages.
## If `s` contains extra data, it will raise `JsonParsingError`.
## This closes the stream `s` after it's done.
## If `rawIntegers` is true, integer literals will not be converted to a `JInt`
## field but kept as raw numbers via `JString`.
## If `rawFloats` is true, floating point literals will not be converted to a `JFloat`
## field but kept as raw numbers via `JString`.
var p: JsonParser
p.open(s, filename)
try:
discard getTok(p) # read first token
result = p.parseJson()
result = p.parseJson(rawIntegers, rawFloats)
eat(p, tkEof) # check if there is no extra data
finally:
p.close()
Expand Down Expand Up @@ -924,6 +960,7 @@ when defined(js):
of JFloat:
result = newJFloat(cast[float](x))
of JString:
# Dunno what to do with isUnquoted here
result = newJString($cast[cstring](x))
of JBool:
result = newJBool(cast[bool](x))
Expand All @@ -937,18 +974,26 @@ when defined(js):
return parseNativeJson(buffer).convertObject()

else:
proc parseJson*(buffer: string): JsonNode =
proc parseJson*(buffer: string; rawIntegers = false, rawFloats = false): JsonNode =
## Parses JSON from `buffer`.
## If `buffer` contains extra data, it will raise `JsonParsingError`.
result = parseJson(newStringStream(buffer), "input")
## If `rawIntegers` is true, integer literals will not be converted to a `JInt`
## field but kept as raw numbers via `JString`.
## If `rawFloats` is true, floating point literals will not be converted to a `JFloat`
## field but kept as raw numbers via `JString`.
result = parseJson(newStringStream(buffer), "input", rawIntegers, rawFloats)

proc parseFile*(filename: string): JsonNode =
proc parseFile*(filename: string; rawIntegers = false, rawFloats = false): JsonNode =
## Parses `file` into a `JsonNode`.
## If `file` contains extra data, it will raise `JsonParsingError`.
## If `rawIntegers` is true, integer literals will not be converted to a `JInt`
## field but kept as raw numbers via `JString`.
## If `rawFloats` is true, floating point literals will not be converted to a `JFloat`
## field but kept as raw numbers via `JString`.
var stream = newFileStream(filename, fmRead)
if stream == nil:
raise newException(IOError, "cannot read from file: " & filename)
result = parseJson(stream, filename)
result = parseJson(stream, filename, rawIntegers, rawFloats)

# -- Json deserialiser. --

Expand Down Expand Up @@ -986,9 +1031,9 @@ when defined(nimFixedForwardGeneric):
proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string)
proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string)
Expand Down Expand Up @@ -1082,7 +1127,7 @@ when defined(nimFixedForwardGeneric):
dst = some(default(T))
initFromJson(dst.get, jsonNode, jsonPath)

macro assignDistinctImpl[T : distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) =
macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) =
let typInst = getTypeInst(dst)
let typImpl = getTypeImpl(dst)
let baseTyp = typImpl[0]
Expand All @@ -1096,7 +1141,7 @@ when defined(nimFixedForwardGeneric):
else:
initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`)

proc initFromJson[T : distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
assignDistinctImpl(dst, jsonNode, jsonPath)

proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode): void =
Expand Down Expand Up @@ -1186,7 +1231,6 @@ when defined(nimFixedForwardGeneric):
else:
error("unhandled kind: " & $typeNode.kind, typeNode)


macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
let typeSym = getTypeInst(dst)
let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen")
Expand All @@ -1201,7 +1245,7 @@ when defined(nimFixedForwardGeneric):
else:
foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen)

proc initFromJson[T : object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) =
assignObjectImpl(dst, jsonNode, jsonPath)

proc to*[T](node: JsonNode, t: typedesc[T]): T =
Expand Down
3 changes: 3 additions & 0 deletions tests/stdlib/tjson.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,3 +232,6 @@ doAssert isRefSkipDistinct(MyRef)
doAssert not isRefSkipDistinct(MyObject)
doAssert isRefSkipDistinct(MyDistinct)
doAssert isRefSkipDistinct(MyOtherDistinct)

let x = parseJson("9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999")
doAssert x.kind == JString

0 comments on commit c1f58d3

Please sign in to comment.