Added knownIssue as key for testament spec.

This is part of a series of commit with intent to re-organize the test suite.

Currently, the 'disabled' key of the testament spec is used both to disable tests on a platform basis and disable buggy tests.
Introducing the 'knownIssue' key in the spec allows simplifying the disabled key, and referencing knownIssues whose behaviour could ( and should ) be revisited.

Further discussion can be found at nim-works#61.

This commit adds a 'knownIssue' key for every test marked with a 'disabled: true' whose disabling could be linked to a knownIssue, be it from git-log messages, or test text.

testament/specs.nim

@@ -104,6 +104,7 @@ type
                       # but don't rely on much precision
     inlineErrors*: seq[InlineError] # line information to error message
     debugInfo*: string # debug info to give more context
+    knownIssues*: seq[string] ## known issues to be fixed
 
 proc getCmd*(s: TSpec): string =
   if s.cmd.len == 0:
@@ -275,7 +276,7 @@ proc parseSpec*(filename: string): TSpec =
     case e.kind
     of cfgKeyValuePair:
       let key = e.key.normalize
-      const allowMultipleOccurences = ["disabled", "ccodecheck"]
+      const allowMultipleOccurences = ["disabled", "ccodecheck" , "knownissue"]
         ## list of flags that are correctly handled when passed multiple times
         ## (instead of being overwritten)
       if key notin allowMultipleOccurences:
@@ -385,7 +386,7 @@ proc parseSpec*(filename: string): TSpec =
         of "netbsd":
           when defined(netbsd): result.err = reDisabled
         else:
-          result.parseErrors.addLine "cannot interpret as a bool: ", e.value
+          result.parseErrors.addLine "cannot interpret as a bool or platform name: ", e.value
       of "cmd":
         if e.value.startsWith("nim "):
           result.cmd = compilerPrefix & e.value[3..^1]
@@ -408,6 +409,12 @@ proc parseSpec*(filename: string): TSpec =
       of "matrix":
         for v in e.value.split(';'):
           result.matrix.add(v.strip)
+      of "knownissue":
+        case e.value.normalize
+        of "n", "no", "false", "0": discard
+        else: 
+            result.knownIssues.add e.value
+            result.err = reDisabled
       else:
         result.parseErrors.addLine "invalid key for test spec: ", e.key
 

tests/concepts/t3330.nim

@@ -1,5 +1,5 @@
 discard """
-disabled: "true"
+knownIssue: "https://github.com/nim-works/nimskull/pull/27"
 description: "explain reporting isn't working during the transition to nkError; revise and ressurrect once fixed"
 errormsg: "type mismatch: got <Bar[system.int]>"
 nimout: '''

tests/concepts/texplain.nim

@@ -1,6 +1,6 @@
 discard """
   cmd: "nim c --verbosity:0 --colors:off $file"
-  disabled: "true"
+  knownIssue: "https://github.com/nim-works/nimskull/pull/27"
   description: "explain reporting isn't working during the transition to nkError; revise and ressurrect once fixed"
   nimout: '''
 texplain.nim(144, 10) Hint: Non-matching candidates for e(y)

tests/concepts/tswizzle.nim

@@ -3,7 +3,7 @@ discard """
 [1, 3]
 [2, 1, 2]
 '''
-  disabled: "true"
+  knownIssue: "https://github.com/nim-lang/Nim/issues/2346"
 """
 
 import macros, strutils

tests/concepts/twrapconcept.nim

@@ -1,5 +1,5 @@
 discard """
-  disabled: "true"
+  knownIssue: "https://github.com/nim-works/nimskull/pull/27"
   description: "concept error reporting isn't working during the transition to nkError; revise and ressurrect once fixed"
   errormsg: "type mismatch: got <string>"
   nimout: "twrapconcept.nim(11, 5) Foo: concept predicate failed"

tests/cpp/tterminate_handler.nim

@@ -2,7 +2,7 @@ discard """
   targets: "cpp"
   outputsub: "Error: unhandled unknown cpp exception"
   exitcode: 1
-  disabled: true
+  knownIssue: "https://github.com/nim-lang/Nim/issues/13695"
 """
 type Crap {.importcpp: "int".} = object
 

tests/cpp/tvectorseq.nim

@@ -2,7 +2,7 @@ discard """
   targets: "cpp"
   output: '''(x: 1.0)
 (x: 0.0)'''
-  disabled: "true"
+  knownIssue: "https://github.com/nim-lang/Nim/issues/2536"
 """
 
 # This cannot work yet because we omit type information for importcpp'ed types.

tests/dll/nimhcr_integration.nim

@@ -1,5 +1,7 @@
 discard """
-  disabled: "true"
+  knownIssue: "https://github.com/nim-lang/Nim/pull/12105"
+  knownIssue: "https://github.com/nim-lang/Nim/pull/17311"
+  knownIssue: "https://github.com/nim-lang/Nim/pull/15458"
   output: '''
 main: HELLO!
 main: hasAnyModuleChanged? true

tests/errmsgs/tundeclared_field.nim

@@ -1,5 +1,5 @@
 discard """
-disabled: "true"
+knownIssue: "https://github.com/nim-works/nimskull/pull/27"
 description: '''
 disabled this test because the ast is being screwed up and we get gensyms when we shouldn't
 

tests/fragmentation/tfragment_alloc.nim

@@ -2,7 +2,9 @@
 discard """
   output: '''occupied ok: true
 total ok: true'''
-  disabled: "true"
+knownIssue: "https://github.com/nim-lang/Nim/issues/8509"
+knownIssue: "https://github.com/nim-lang/Nim/issues/7120"
+knownIssue: "https://github.com/nim-lang/Nim/issues/9421"
 """
 
 import strutils, data

tests/fragmentation/tfragment_gc.nim

@@ -1,7 +1,9 @@
 discard """
   output: '''occupied ok: true
 total ok: true'''
-  disabled: "true"
+knownIssue: "https://github.com/nim-lang/Nim/issues/8509"
+knownIssue: "https://github.com/nim-lang/Nim/issues/7120"
+knownIssue: "https://github.com/nim-lang/Nim/issues/9421"
 """
 
 import strutils, data

tests/iter/tchainediterators.nim

@@ -6,7 +6,7 @@ discard """
 128
 192
 '''
-  disabled: "true"
+  knownIssue: "https://github.com/nim-lang/Nim/issues/1838"
 """
 
 # This all relies on non-documented and questionable features.

tests/macros/tmacro7.nim

@@ -3,7 +3,7 @@ output: '''
 calling!stuff
 calling!stuff
 '''
-disabled: true
+knownIssue: "https://github.com/nim-lang/Nim/issues/11883"
 """
 
 # this test modifies an already semchecked ast (bad things happen)

tests/misc/tnoinst.nim

@@ -1,7 +1,8 @@
 discard """
   errormsg: "instantiate 'notConcrete' explicitly"
   line: 12
-  disabled: "true"
+  knownIssue: "https://github.com/nim-lang/Nim/issues/1708"
+  knownIssue: "https://github.com/nim-lang/Nim/issues/871"
 """
 
 proc wrap[T]() =

tests/objvariant/tfloatrangeobj.nim

@@ -2,7 +2,8 @@ discard """
   output: '''(kind: 2.0, twoStr: "TWO STR")
 (kind: 1.0)
 '''
-disabled: "true"
+knownIssue: "https://github.com/nim-lang/Nim/issues/12379"
+knownIssue: "https://github.com/nim-lang/Nim/issues/12591"
 """
 type
   FloatRange = range[1.0..3.0]

tests/proc/t17157.nim

@@ -1,6 +1,7 @@
 discard """
   errormsg: "'untyped' is only allowed in templates and macros or magic procs"
-  disabled: true
+  knownIssue: "https://github.com/nim-lang/Nim/issues/18113"
+  knownIssue: "https://github.com/nim-lang/Nim/issues/18124"
 """
 
 template something(op: proc (v: untyped): void): void =