test nimsuggest on travis; add disabled:true|false to disable currently failing tests

.travis.yml

@@ -50,4 +50,4 @@ script:
   - ./koch web
   - ./koch csource
   - ./koch nimsuggest
-#  - nim c -r nimsuggest/tester
+  - nim c -r nimsuggest/tester

nimsuggest/tester.nim

@@ -7,9 +7,10 @@ import os, osproc, strutils, streams, re, sexp, net
 
 type
   Test = object
-    cmd, dest: string
+    filename, cmd, dest: string
     startup: seq[string]
     script: seq[(string, string)]
+    disabled: bool
 
 const
   curDir = when defined(windows): "" else: ""
@@ -21,6 +22,7 @@ proc parseTest(filename: string; epcMode=false): Test =
   const cursorMarker = "#[!]#"
   let nimsug = curDir & addFileExt("nimsuggest", ExeExt)
   let libpath = findExe("nim").splitFile().dir /../ "lib"
+  result.filename = filename
   result.dest = getTempDir() / extractFilename(filename)
   result.cmd = nimsug & " --tester " & result.dest
   result.script = @[]
@@ -42,7 +44,14 @@ proc parseTest(filename: string; epcMode=false): Test =
     if x.contains("""""""""):
       inc specSection
     elif specSection == 1:
-      if x.startsWith("$nimsuggest"):
+      if x.startsWith("disabled:"):
+        if x.startsWith("disabled:true"):
+          result.disabled = true
+        else:
+          # be strict about format
+          doAssert x.startsWith("disabled:false")
+          result.disabled = false
+      elif x.startsWith("$nimsuggest"):
         result.cmd = x % ["nimsuggest", nimsug, "file", filename, "lib", libpath]
       elif x.startsWith("!"):
         if result.cmd.len == 0:
@@ -223,8 +232,14 @@ proc doReport(filename, answer, resp: string; report: var string) =
       report.add "\n  Expected:  " & resp
       report.add "\n  But got:   " & answer
 
+proc skipDisabledTest(test: Test): bool =
+  if test.disabled:
+    echo "disabled: " & test.filename
+  result = test.disabled
+
 proc runEpcTest(filename: string): int =
   let s = parseTest(filename, true)
+  if s.skipDisabledTest: return 0
   for cmd in s.startup:
     if not runCmd(cmd, s.dest):
       quit "invalid command: " & cmd
@@ -267,6 +282,7 @@ proc runEpcTest(filename: string): int =
 
 proc runTest(filename: string): int =
   let s = parseTest filename
+  if s.skipDisabledTest: return 0
   for cmd in s.startup:
     if not runCmd(cmd, s.dest):
       quit "invalid command: " & cmd

nimsuggest/tests/tchk1.nim

@@ -15,6 +15,7 @@ proc main =
 
 #[!]#
 discard """
+disabled:true
 $nimsuggest --tester $file
 >chk $1
 chk;;skUnknown;;;;Hint;;???;;-1;;-1;;"tchk1 [Processing]";;0

nimsuggest/tests/tdot4.nim

@@ -1,4 +1,5 @@
 discard """
+disabled:true
 $nimsuggest --tester --maxresults:2 $file
 >sug $1
 sug;;skProc;;tdot4.main;;proc (inp: string): string;;$file;;10;;5;;"";;100;;None

nimsuggest/tests/tinclude.nim

@@ -1,4 +1,5 @@
 discard """
+disabled:true
 $nimsuggest --tester compiler/nim.nim
 >def compiler/semexprs.nim:25:50
 def;;skType;;ast.PSym;;PSym;;*ast.nim;;707;;2;;"";;100

nimsuggest/tests/tstrutils.nim

@@ -1,4 +1,5 @@
 discard """
+disabled:true
 $nimsuggest --tester lib/pure/strutils.nim
 >def lib/pure/strutils.nim:2529:6
 def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"same as `assert` but is always turned on and not affected by the\x0A``--assertions`` command line switch.";;100

nimsuggest/tests/tsug_regression.nim

@@ -17,6 +17,7 @@ proc main =
   map0.#[!]#
 
 discard """
+disabled:true
 $nimsuggest --tester $file
 >sug $1
 sug;;skProc;;tables.getOrDefault;;proc (t: Table[getOrDefault.A, getOrDefault.B], key: A): B;;$lib/pure/collections/tables.nim;;178;;5;;"";;100;;None

nimsuggest/tests/ttype_decl.nim

@@ -1,4 +1,5 @@
 discard """
+disabled:true
 $nimsuggest --tester --maxresults:3 $file
 >sug $1
 sug;;skType;;ttype_decl.Other;;Other;;$file;;10;;2;;"";;0;;None

nimsuggest/tests/twithin_macro.nim

@@ -202,6 +202,7 @@ echo r.age_human_yrs()
 echo r
 
 discard """
+disabled:true
 $nimsuggest --tester $file
 >sug $1
 sug;;skField;;age;;int;;$file;;167;;6;;"";;100;;None

nimsuggest/tests/twithin_macro_prefix.nim

@@ -202,6 +202,7 @@ echo r.age_human_yrs()
 echo r
 
 discard """
+disabled:true
 $nimsuggest --tester $file
 >sug $1
 sug;;skField;;age;;int;;$file;;167;;6;;"";;100;;Prefix