StringStream & more stdlib modules support for JS/NimScript (nim-lang…

…#14095)

* StringStream & more stdlib modules support for JS/NimScript

* change back pegs test in line with nim-lang#14134

changelog.md

@@ -37,6 +37,17 @@
   and this can now throw in edge cases where `getCurrentDir` throws.
   `relativePath` also now works for js with `-d:nodejs`.
 
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`. 
+
 - Added `streams.readStr` and `streams.peekStr` overloads to
   accept an existing string to modify, which avoids memory
   allocations, similar to `streams.readLine` (#13857).

doc/backends.rst

@@ -76,13 +76,15 @@ available. This includes:
 * manual memory management (``alloc``, etc.)
 * casting and other unsafe operations (``cast`` operator, ``zeroMem``, etc.)
 * file management
-* most modules of the standard library
+* OS-specific operations
+* threading, coroutines
+* some modules of the standard library
 * proper 64 bit integer arithmetic
-* unsigned integer arithmetic
 
-However, the modules `strutils <strutils.html>`_, `math <math.html>`_, and
-`times <times.html>`_ are available! To access the DOM, use the `dom
-<dom.html>`_ module that is only available for the JavaScript platform.
+To compensate, the standard library has modules `catered to the JS backend
+<https://nim-lang.org/docs/lib.html#pure-libraries-modules-for-js-backend>`_
+and more support will come in the future (for instance, Node.js bindings
+to get OS info).
 
 To compile a Nim module into a ``.js`` file use the ``js`` command; the
 default is a ``.js`` file that is supposed to be referenced in an ``.html``

doc/nims.rst

@@ -52,8 +52,6 @@ NimScript is subject to some limitations caused by the implementation of the VM
 
 * ``random.randomize()`` requires an ``int64`` explicitly passed as argument, you *must* pass a Seed integer.
 
-* ``unicode`` can be imported, but not ``unidecode``.
-
 
 Standard library modules
 ========================

lib/pure/cstrutils.nim

@@ -19,29 +19,43 @@ proc toLowerAscii(c: char): char {.inline.} =
   else:
     result = c
 
-proc startsWith*(s, prefix: cstring): bool {.noSideEffect,
-  rtl, extern: "csuStartsWith".} =
-  ## Returns true iff ``s`` starts with ``prefix``.
-  ##
-  ## If ``prefix == ""`` true is returned.
-  var i = 0
-  while true:
-    if prefix[i] == '\0': return true
-    if s[i] != prefix[i]: return false
-    inc(i)
+when defined(js):
+  proc startsWith*(s, prefix: cstring): bool {.noSideEffect,
+    importjs: "#.startsWith(#)".}
 
-proc endsWith*(s, suffix: cstring): bool {.noSideEffect,
-  rtl, extern: "csuEndsWith".} =
-  ## Returns true iff ``s`` ends with ``suffix``.
-  ##
-  ## If ``suffix == ""`` true is returned.
-  let slen = s.len
-  var i = 0
-  var j = slen - len(suffix)
-  while i+j <% slen:
-    if s[i+j] != suffix[i]: return false
-    inc(i)
-  if suffix[i] == '\0': return true
+  proc endsWith*(s, suffix: cstring): bool {.noSideEffect,
+    importjs: "#.endsWith(#)".}
+
+  # JS string has more operations that might warrant its own module:
+  # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String
+else:
+  proc startsWith*(s, prefix: cstring): bool {.noSideEffect,
+    rtl, extern: "csuStartsWith".} =
+    ## Returns true iff ``s`` starts with ``prefix``.
+    ##
+    ## If ``prefix == ""`` true is returned.
+    ## 
+    ## JS backend uses native ``String.prototype.startsWith``.
+    var i = 0
+    while true:
+      if prefix[i] == '\0': return true
+      if s[i] != prefix[i]: return false
+      inc(i)
+
+  proc endsWith*(s, suffix: cstring): bool {.noSideEffect,
+    rtl, extern: "csuEndsWith".} =
+    ## Returns true iff ``s`` ends with ``suffix``.
+    ##
+    ## If ``suffix == ""`` true is returned.
+    ## 
+    ## JS backend uses native ``String.prototype.endsWith``.
+    let slen = s.len
+    var i = 0
+    var j = slen - len(suffix)
+    while i+j <% slen:
+      if s[i+j] != suffix[i]: return false
+      inc(i)
+    if suffix[i] == '\0': return true
 
 proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect,
   rtl, extern: "csuCmpIgnoreStyle".} =
@@ -53,6 +67,9 @@ proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect,
   ## | 0 iff a == b
   ## | < 0 iff a < b
   ## | > 0 iff a > b
+  ## 
+  ## Not supported for JS backend, use `strutils.cmpIgnoreStyle
+  ## <https://nim-lang.org/docs/strutils.html#cmpIgnoreStyle%2Cstring%2Cstring>`_ instead.
   var i = 0
   var j = 0
   while true:
@@ -72,6 +89,9 @@ proc cmpIgnoreCase*(a, b: cstring): int {.noSideEffect,
   ## | 0 iff a == b
   ## | < 0 iff a < b
   ## | > 0 iff a > b
+  ## 
+  ## Not supported for JS backend, use `strutils.cmpIgnoreCase
+  ## <https://nim-lang.org/docs/strutils.html#cmpIgnoreCase%2Cstring%2Cstring>`_ instead.
   var i = 0
   while true:
     var aa = toLowerAscii(a[i])

lib/pure/lexbase.nim

@@ -52,7 +52,8 @@ proc fillBuffer(L: var BaseLexer) =
   toCopy = L.buf.len - (L.sentinel + 1)
   assert(toCopy >= 0)
   if toCopy > 0:
-    when defined(js):
+    when defined(js) or defined(nimscript):
+      # nimscript has to be here to avoid compiling other branch (moveMem)
       for i in 0 ..< toCopy:
         L.buf[i] = L.buf[L.sentinel + 1 + i]
     else:

lib/pure/marshal.nim

@@ -50,8 +50,14 @@
 ## * `streams module <streams.html>`_
 ## * `json module <json.html>`_
 
-when defined(nimV2):
-  {.error: """marshal module is not supported in new runtime.
+const unsupportedPlatform =
+  when defined(nimV2): "new runtime"
+  elif defined(js): "javascript"
+  elif defined(nimscript): "nimscript"
+  else: ""
+
+when unsupportedPlatform != "":
+  {.error: "marshal module is not supported in " & unsupportedPlatform & """.
 Please use alternative packages for serialization. 
 It is possible to reimplement this module using generics and type traits. 
 Please contribute new implementation.""".}

lib/pure/parsesql.nim

@@ -557,7 +557,14 @@ type
     tok: Token
 
 proc newNode*(k: SqlNodeKind): SqlNode =
-  result = SqlNode(kind: k)
+  when defined(js): # bug #14117
+    case k
+    of LiteralNodes:
+      result = SqlNode(kind: k, strVal: "")
+    else:
+      result = SqlNode(kind: k, sons: @[])
+  else:
+    result = SqlNode(kind: k)
 
 proc newNode*(k: SqlNodeKind, s: string): SqlNode =
   result = SqlNode(kind: k)
@@ -1469,34 +1476,33 @@ proc treeRepr*(s: SqlNode): string =
   result = newStringOfCap(128)
   treeReprAux(s, 0, result)
 
-when not defined(js):
-  import streams
+import streams
 
-  proc open(L: var SqlLexer, input: Stream, filename: string) =
-    lexbase.open(L, input)
-    L.filename = filename
+proc open(L: var SqlLexer, input: Stream, filename: string) =
+  lexbase.open(L, input)
+  L.filename = filename
 
-  proc open(p: var SqlParser, input: Stream, filename: string) =
-    ## opens the parser `p` and assigns the input stream `input` to it.
-    ## `filename` is only used for error messages.
-    open(SqlLexer(p), input, filename)
-    p.tok.kind = tkInvalid
-    p.tok.literal = ""
-    getTok(p)
+proc open(p: var SqlParser, input: Stream, filename: string) =
+  ## opens the parser `p` and assigns the input stream `input` to it.
+  ## `filename` is only used for error messages.
+  open(SqlLexer(p), input, filename)
+  p.tok.kind = tkInvalid
+  p.tok.literal = ""
+  getTok(p)
 
-  proc parseSQL*(input: Stream, filename: string): SqlNode =
-    ## parses the SQL from `input` into an AST and returns the AST.
-    ## `filename` is only used for error messages.
-    ## Syntax errors raise an `SqlParseError` exception.
-    var p: SqlParser
-    open(p, input, filename)
-    try:
-      result = parse(p)
-    finally:
-      close(p)
-
-  proc parseSQL*(input: string, filename = ""): SqlNode =
-    ## parses the SQL from `input` into an AST and returns the AST.
-    ## `filename` is only used for error messages.
-    ## Syntax errors raise an `SqlParseError` exception.
-    parseSQL(newStringStream(input), "")
+proc parseSQL*(input: Stream, filename: string): SqlNode =
+  ## parses the SQL from `input` into an AST and returns the AST.
+  ## `filename` is only used for error messages.
+  ## Syntax errors raise an `SqlParseError` exception.
+  var p: SqlParser
+  open(p, input, filename)
+  try:
+    result = parse(p)
+  finally:
+    close(p)
+
+proc parseSQL*(input: string, filename = ""): SqlNode =
+  ## parses the SQL from `input` into an AST and returns the AST.
+  ## `filename` is only used for error messages.
+  ## Syntax errors raise an `SqlParseError` exception.
+  parseSQL(newStringStream(input), "")

lib/pure/ropes.nim

@@ -287,46 +287,47 @@ proc addf*(c: var Rope, frmt: string, args: openArray[Rope]) {.
   ## shortcut for ``add(c, frmt % args)``.
   add(c, frmt % args)
 
-const
-  bufSize = 1024 # 1 KB is reasonable
-
-proc equalsFile*(r: Rope, f: File): bool {.rtl, extern: "nro$1File".} =
-  ## returns true if the contents of the file `f` equal `r`.
-  var
-    buf: array[bufSize, char]
-    bpos = buf.len
-    blen = buf.len
-
-  for s in leaves(r):
-    var spos = 0
-    let slen = s.len
-    while spos < slen:
-      if bpos == blen:
-        # Read more data
-        bpos = 0
-        blen = readBuffer(f, addr(buf[0]), buf.len)
-        if blen == 0: # no more data in file
+when not defined(js) and not defined(nimscript):
+  const
+    bufSize = 1024 # 1 KB is reasonable
+
+  proc equalsFile*(r: Rope, f: File): bool {.rtl, extern: "nro$1File".} =
+    ## returns true if the contents of the file `f` equal `r`.
+    var
+      buf: array[bufSize, char]
+      bpos = buf.len
+      blen = buf.len
+
+    for s in leaves(r):
+      var spos = 0
+      let slen = s.len
+      while spos < slen:
+        if bpos == blen:
+          # Read more data
+          bpos = 0
+          blen = readBuffer(f, addr(buf[0]), buf.len)
+          if blen == 0: # no more data in file
+            result = false
+            return
+        let n = min(blen - bpos, slen - spos)
+        # TODO There's gotta be a better way of comparing here...
+        if not equalMem(addr(buf[bpos]),
+                        cast[pointer](cast[int](cstring(s))+spos), n):
           result = false
           return
-      let n = min(blen - bpos, slen - spos)
-      # TODO There's gotta be a better way of comparing here...
-      if not equalMem(addr(buf[bpos]),
-                      cast[pointer](cast[int](cstring(s))+spos), n):
-        result = false
-        return
-      spos += n
-      bpos += n
-
-  result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all
-
-proc equalsFile*(r: Rope, filename: string): bool {.rtl, extern: "nro$1Str".} =
-  ## returns true if the contents of the file `f` equal `r`. If `f` does not
-  ## exist, false is returned.
-  var f: File
-  result = open(f, filename)
-  if result:
-    result = equalsFile(r, f)
-    close(f)
+        spos += n
+        bpos += n
+
+    result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all
+
+  proc equalsFile*(r: Rope, filename: string): bool {.rtl, extern: "nro$1Str".} =
+    ## returns true if the contents of the file `f` equal `r`. If `f` does not
+    ## exist, false is returned.
+    var f: File
+    result = open(f, filename)
+    if result:
+      result = equalsFile(r, f)
+      close(f)
 
 new(N) # init dummy node for splay algorithm