tools: support testing known issues

This commit adds a known_issues directory to the test directory
for scripts that reproduce known bugs. Since these scripts are
expected to fail, it also adds a --expect-fail flag to test.py
which reports tests as successful when they fail.

Refs: nodejs/testing#18
Backport-URL: #5785
PR-URL: #5528
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Wyatt Preul <wpreul@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

Conflicts:
	tools/test.py

Makefile

@@ -168,6 +168,9 @@ test-internet: all
 test-debugger: all
 	$(PYTHON) tools/test.py debugger
 
+test-known-issues: all
+	$(PYTHON) tools/test.py known_issues --expect-fail
+
 test-npm: $(NODE_EXE)
 	NODE=$(NODE) tools/test-npm.sh
 

test/known_issues/testcfg.py

@@ -0,0 +1,6 @@
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import testpy
+
+def GetConfiguration(context, root):
+  return testpy.SimpleTestConfiguration(context, root, 'known_issues')

tools/test.py

@@ -430,7 +430,7 @@ def __init__(self, context, path, arch, mode):
     self.thread_id = 0
 
   def IsNegative(self):
-    return False
+    return self.context.expect_fail
 
   def CompareTime(self, other):
     return cmp(other.duration, self.duration)
@@ -774,11 +774,13 @@ def GetTestStatus(self, context, sections, defs):
 
 class Context(object):
 
-  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output):
+  def __init__(self, workspace, buildspace, verbose, vm, expect_fail,
+               timeout, processor, suppress_dialogs, store_unexpected_output):
     self.workspace = workspace
     self.buildspace = buildspace
     self.verbose = verbose
     self.vm_root = vm
+    self.expect_fail = expect_fail
     self.timeout = timeout
     self.processor = processor
     self.suppress_dialogs = suppress_dialogs
@@ -1281,6 +1283,8 @@ def BuildOptions():
   result.add_option("--snapshot", help="Run the tests with snapshot turned on",
       default=False, action="store_true")
   result.add_option("--special-command", default=None)
+  result.add_option("--expect-fail", dest="expect_fail",
+      help="Expect test cases to fail", default=False, action="store_true")
   result.add_option("--valgrind", help="Run tests through valgrind",
       default=False, action="store_true")
   result.add_option("--cat", help="Print the source of the tests",
@@ -1471,6 +1475,7 @@ def Main():
                     buildspace,
                     VERBOSE,
                     shell,
+                    options.expect_fail,
                     options.timeout,
                     processor,
                     options.suppress_dialogs,

vcbuild.bat

@@ -61,6 +61,7 @@ if /i "%1"=="test-gc"       set test_args=%test_args% gc&set buildnodeweak=1&got
 if /i "%1"=="test-internet" set test_args=%test_args% internet&goto arg-ok
 if /i "%1"=="test-pummel"   set test_args=%test_args% pummel&goto arg-ok
 if /i "%1"=="test-all"      set test_args=%test_args% sequential parallel message gc internet pummel&set buildnodeweak=1&set jslint=1&goto arg-ok
+if /i "%1"=="test-known-issues" set test_args=%test_args% known_issues --expect-fail&goto arg-ok
 if /i "%1"=="jslint"        set jslint=1&goto arg-ok
 if /i "%1"=="msi"           set msi=1&set licensertf=1&set download_arg="--download=all"&set i18n_arg=small-icu&goto arg-ok
 if /i "%1"=="build-release" set build_release=1&goto arg-ok