new gensym handling

changelog.md

@@ -10,6 +10,13 @@
   to UTF-8. Use the new switch `-d:nimDontSetUtf8CodePage` to disable this
   feature.
 
+- The language definition and compiler are now stricter about ``gensym``'ed
+  symbols in hygienic templates. See the section in the
+  [manual](https://nim-lang.org/docs/manual.html#templates-hygiene-in-templates)
+  for further details. Use the compiler switch `--useVersion:0.19` for a
+  transition period.
+
+
 ### Breaking changes in the standard library
 
 

compiler/commands.nim

@@ -788,6 +788,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "expandmacro":
     expectArg(conf, switch, arg, pass, info)
     conf.macrosToExpand[arg] = "T"
+  of "useversion":
+    expectArg(conf, switch, arg, pass, info)
+    case arg
+    of "0.19":
+      conf.globalOptions.incl optNimV019
+    of "1.0":
+      discard "the default"
+    else:
+      localError(conf, info, "unknown Nim version; currently supported values are: {0.19, 1.0}")
   of "":
     conf.projectName = "-"
   else:

compiler/evaltempl.nim

@@ -10,7 +10,7 @@
 ## Template evaluation engine. Now hygienic.
 
 import
-  strutils, options, ast, astalgo, msgs, renderer, lineinfos
+  strutils, options, ast, astalgo, msgs, renderer, lineinfos, idents
 
 type
   TemplCtx = object
@@ -20,6 +20,7 @@ type
     mapping: TIdTable # every gensym'ed symbol needs to be mapped to some
                       # new symbol
     config: ConfigRef
+    ic: IdentCache
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
@@ -52,7 +53,11 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
           #if x.kind == skParam and x.owner.kind == skModule:
           #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
-        result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
+        if sfGenSym in s.flags and optNimV019 notin c.config.globalOptions:
+          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $x.id),
+            if c.instLines: actual.info else: templ.info)
+        else:
+          result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
     else:
       result.add copyNode(c, templ, actual)
   of nkNone..nkIdent, nkType..nkNilLit: # atom
@@ -160,7 +165,9 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
     result.typ = res.typ
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
-                   conf: ConfigRef; fromHlo=false): PNode =
+                   conf: ConfigRef;
+                   ic: IdentCache;
+                   fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
     globalError(conf, n.info, errTemplateInstantiationTooNested)
@@ -172,6 +179,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.owner = tmpl
   ctx.genSymOwner = genSymOwner
   ctx.config = conf
+  ctx.ic = ic
   initIdTable(ctx.mapping)
 
   let body = tmpl.getBody

compiler/main.nim

@@ -18,7 +18,7 @@ import
   sem, idents, passes, extccomp,
   cgen, json, nversion,
   platform, nimconf, passaux, depends, vm, idgen,
-  parser, modules,
+  modules,
   modulegraphs, tables, rod, lineinfos, pathutils
 
 when not defined(leanCompiler):

compiler/options.nim

@@ -84,6 +84,7 @@ type                          # please make sure we have under 32 options
     optDynlibOverrideAll
     optNimV2
     optMultiMethods
+    optNimV019
 
   TGlobalOptions* = set[TGlobalOption]
 

compiler/semexprs.nim

@@ -30,7 +30,7 @@ proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
   # Note: This is n.info on purpose. It prevents template from creating an info
   # context when called from an another template
   pushInfoContext(c.config, n.info, s.detailedInfo)
-  result = evalTemplate(n, s, getCurrOwner(c), c.config, efFromHlo in flags)
+  result = evalTemplate(n, s, getCurrOwner(c), c.config, c.cache, efFromHlo in flags)
   if efNoSemCheck notin flags: result = semAfterMacroCall(c, n, result, s, flags)
   popInfoContext(c.config)
 
@@ -1236,6 +1236,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     result = newSymNode(s, n.info)
   else:
     let info = getCallLineInfo(n)
+    #if efInCall notin flags:
     markUsed(c, info, s)
     onUse(info, s)
     result = newSymNode(s, info)

compiler/semtempl.nim

@@ -47,7 +47,8 @@ type
   TSymChoiceRule = enum
     scClosed, scOpen, scForceOpen
 
-proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode =
+proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule;
+               isField = false): PNode =
   var
     a: PSym
     o: TOverloadIter
@@ -63,9 +64,12 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode =
     # XXX this makes more sense but breaks bootstrapping for now:
     # (s.kind notin routineKinds or s.magic != mNone):
     # for instance 'nextTry' is both in tables.nim and astalgo.nim ...
-    result = newSymNode(s, info)
-    markUsed(c, info, s)
-    onUse(info, s)
+    if not isField or sfGenSym notin s.flags:
+      result = newSymNode(s, info)
+      markUsed(c, info, s)
+      onUse(info, s)
+    else:
+      result = n
   else:
     # semantic checking requires a type; ``fitNode`` deals with it
     # appropriately
@@ -74,7 +78,7 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode =
     result = newNodeIT(kind, info, newTypeS(tyNone, c))
     a = initOverloadIter(o, c, n)
     while a != nil:
-      if a.kind != skModule:
+      if a.kind != skModule and (not isField or sfGenSym notin s.flags):
         incl(a.flags, sfUsed)
         addSon(result, newSymNode(a, info))
         onUse(info, a)
@@ -119,6 +123,7 @@ type
     owner: PSym
     cursorInBody: bool # only for nimsuggest
     scopeN: int
+    noGenSym: int
 
 template withBracketExpr(ctx, x, body: untyped) =
   body
@@ -228,7 +233,7 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
     else:
       replaceIdentBySym(c.c, n, ident)
 
-proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode =
+proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
   incl(s.flags, sfUsed)
   # we do not call onUse here, as the identifier is not really
   # resolved here. We will fixup the used identifiers later.
@@ -237,15 +242,18 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode =
     # Introduced in this pass! Leave it as an identifier.
     result = n
   of OverloadableSyms:
-    result = symChoice(c, n, s, scOpen)
+    result = symChoice(c, n, s, scOpen, isField)
   of skGenericParam:
-    result = newSymNodeTypeDesc(s, n.info)
+    if isField: result = n
+    else: result = newSymNodeTypeDesc(s, n.info)
   of skParam:
     result = n
   of skType:
-    result = newSymNodeTypeDesc(s, n.info)
+    if isField: result = n
+    else: result = newSymNodeTypeDesc(s, n.info)
   else:
-    result = newSymNode(s, n.info)
+    if isField: result = n
+    else: result = newSymNode(s, n.info)
 
 proc semRoutineInTemplName(c: var TemplCtx, n: PNode): PNode =
   result = n
@@ -322,22 +330,23 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
     if n.ident.id in c.toInject: return n
     let s = qualifiedLookUp(c.c, n, {})
     if s != nil:
-      if s.owner == c.owner and s.kind == skParam:
+      if s.owner == c.owner and s.kind == skParam and
+          (sfGenSym notin s.flags or c.noGenSym == 0):
         incl(s.flags, sfUsed)
         result = newSymNode(s, n.info)
         onUse(n.info, s)
       elif contains(c.toBind, s.id):
-        result = symChoice(c.c, n, s, scClosed)
+        result = symChoice(c.c, n, s, scClosed, c.noGenSym > 0)
       elif contains(c.toMixin, s.name.id):
-        result = symChoice(c.c, n, s, scForceOpen)
-      elif s.owner == c.owner and sfGenSym in s.flags:
+        result = symChoice(c.c, n, s, scForceOpen, c.noGenSym > 0)
+      elif s.owner == c.owner and sfGenSym in s.flags and c.noGenSym == 0:
         # template tmp[T](x: var seq[T]) =
         # var yz: T
         incl(s.flags, sfUsed)
         result = newSymNode(s, n.info)
         onUse(n.info, s)
       else:
-        result = semTemplSymbol(c.c, n, s)
+        result = semTemplSymbol(c.c, n, s, c.noGenSym > 0)
   of nkBind:
     result = semTemplBody(c, n.sons[0])
   of nkBindStmt:
@@ -524,12 +533,27 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
         onUse(n.info, s)
         return newSymNode(s, n.info)
       elif contains(c.toBind, s.id):
-        return symChoice(c.c, n, s, scClosed)
+        return symChoice(c.c, n, s, scClosed, c.noGenSym > 0)
       elif contains(c.toMixin, s.name.id):
-        return symChoice(c.c, n, s, scForceOpen)
+        return symChoice(c.c, n, s, scForceOpen, c.noGenSym > 0)
       else:
-        return symChoice(c.c, n, s, scOpen)
-    result = semTemplBodySons(c, n)
+        return symChoice(c.c, n, s, scOpen, c.noGenSym > 0)
+    if n.kind == nkDotExpr:
+      result = n
+      result.sons[0] = semTemplBody(c, n.sons[0])
+      inc c.noGenSym
+      result.sons[1] = semTemplBody(c, n.sons[1])
+      dec c.noGenSym
+    else:
+      result = semTemplBodySons(c, n)
+  of nkExprColonExpr, nkExprEqExpr:
+    if n.len == 2:
+      inc c.noGenSym
+      result.sons[0] = semTemplBody(c, n.sons[0])
+      dec c.noGenSym
+      result.sons[1] = semTemplBody(c, n.sons[1])
+    else:
+      result = semTemplBodySons(c, n)
   else:
     result = semTemplBodySons(c, n)
 

compiler/suggest.nim

@@ -261,15 +261,15 @@ proc getQuality(s: PSym): range[0..100] =
     if exp.kind in {tyUntyped, tyTyped, tyGenericParam, tyAnything}: return 50
   return 100
 
-template wholeSymTab(cond, section: untyped) =
+template wholeSymTab(cond, section: untyped) {.dirty.} =
   var isLocal = true
   var scopeN = 0
   for scope in walkScopes(c.currentScope):
     if scope == c.topLevelScope: isLocal = false
     dec scopeN
     for item in scope.symbols:
-      let it {.inject.} = item
-      var pm {.inject.}: PrefixMatch
+      let it = item
+      var pm: PrefixMatch
       if cond:
         outputs.add(symToSuggest(c.config, it, isLocal = isLocal, section, info, getQuality(it),
                                  pm, c.inTypeContext > 0, scopeN))

compiler/vm.nim

@@ -1137,7 +1137,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           let node = regs[rb+i].regToNode
           node.info = c.debug[pc]
           macroCall.add(node)
-        var a = evalTemplate(macroCall, prc, genSymOwner, c.config)
+        var a = evalTemplate(macroCall, prc, genSymOwner, c.config, c.cache)
         if a.kind == nkStmtList and a.len == 1: a = a[0]
         a.recSetFlagIsRef
         ensureKind(rkNode)

doc/advopt.txt

@@ -122,7 +122,7 @@ Advanced options:
                             enable experimental language feature
   --legacy:$2
                             enable obsolete/legacy language feature
-                            legacy code.
+  --useVersion:0.19|1.0     emulate Nim version X of the Nim compiler
   --newruntime              use an alternative runtime that uses destructors
                             and that uses a shared heap via -d:useMalloc
   --profiler:on|off         enable profiling; requires `import nimprof`, and

doc/manual.rst

@@ -4893,6 +4893,45 @@ no semantics outside of a template definition and cannot be abstracted over:
 To get rid of hygiene in templates, one can use the `dirty`:idx: pragma for
 a template. ``inject`` and ``gensym`` have no effect in ``dirty`` templates.
 
+``gensym``'ed symbols cannot be used as ``field`` in the ``x.field`` syntax.
+Nor can they be used in the ``ObjectConstruction(field: value)``
+and ``namedParameterCall(field = value)`` syntactic constructs.
+
+The reason for this is that code like
+
+.. code-block:: nim
+    :test: "nim c $1"
+
+  type
+    T = object
+      f: int
+
+  template tmp(x: T) =
+    let f = 34
+    echo x.f, T(f: 4)
+
+
+should work as expected.
+
+However, this means that the method call syntax is not available for
+``gensym``'ed symbols:
+
+.. code-block:: nim
+    :test: "nim c $1"
+    :status: 1
+
+  template tmp(x) =
+    type
+      T {.gensym.} = int
+
+    echo x.T # invalid: instead use:  'echo T(x)'.
+
+  tmp(12)
+
+
+**Note**: The Nim compiler prior to version 1 was more lenient about this
+requirement. Use the ``--useVersion:0.19`` switch for a transition period.
+
 
 
 Limitations of the method call syntax

lib/core/locks.nim

@@ -60,11 +60,11 @@ template withLock*(a: Lock, body: untyped) =
   ## Acquires the given lock, executes the statements in body and
   ## releases the lock after the statements finish executing.
   mixin acquire, release
-  a.acquire()
+  acquire(a)
   {.locks: [a].}:
     try:
       body
     finally:
-      a.release()
+      release(a)
 
 {.pop.}

lib/system/alloc.nim

@@ -984,7 +984,7 @@ when defined(nimTypeNames):
 
 # ---------------------- thread memory region -------------------------------
 
-template instantiateForRegion(allocator: untyped) =
+template instantiateForRegion(allocator: untyped) {.dirty.} =
   {.push stackTrace: off.}
 
   when defined(fulldebug):
@@ -1006,8 +1006,8 @@ template instantiateForRegion(allocator: untyped) =
   proc dealloc(p: pointer) =
     dealloc(allocator, p)
 
-  proc realloc(p: pointer, newsize: Natural): pointer =
-    result = realloc(allocator, p, newsize)
+  proc realloc(p: pointer, newSize: Natural): pointer =
+    result = realloc(allocator, p, newSize)
 
   when false:
     proc countFreeMem(): int =
@@ -1054,13 +1054,13 @@ template instantiateForRegion(allocator: untyped) =
     else:
       dealloc(p)
 
-  proc reallocShared(p: pointer, newsize: Natural): pointer =
+  proc reallocShared(p: pointer, newSize: Natural): pointer =
     when hasThreadSupport:
       acquireSys(heapLock)
-      result = realloc(sharedHeap, p, newsize)
+      result = realloc(sharedHeap, p, newSize)
       releaseSys(heapLock)
     else:
-      result = realloc(p, newsize)
+      result = realloc(p, newSize)
 
   when hasThreadSupport:
     template sharedMemStatsShared(v: int) =