assertion count, custom test manager

* Fix assertion count by skipping log messages.
* Add suport for using a custom test manager by defining it through the TEST_MANAGER env.

ci/Runner.cls

@@ -7,21 +7,22 @@ ClassMethod Run(configuration As CI.Configuration) As %Status
 {
   new $namespace
 
-  set environmentNS = configuration.GetEnv("CI_NAMESPACE")
+  set environmentNS = configuration.GetEnv("CI_NAMESPACE")  
+  set testManager = configuration.GetEnv("TEST_MANAGER")
   set $namespace = $get(environmentNS, $namespace)
 
   do ..ResolveConfiguration(configuration, .testSpecs, .flags, .params)
   do ..DescribeConfiguration(testSpecs, flags)
 
-  return ..RunSilently(testSpecs, flags, .params)
+  return ..RunSilently(testManager, testSpecs, flags, .params)
 }
 
 ClassMethod ResolveConfiguration(configuration As CI.Configuration, Output testSpecs As %String, Output flags As %String, Output params As %String)
 {
   do configuration.GetTestParameters(.params)
 
   set ^UnitTestRoot = "/opt/ci/app"
-  set flags = "/"_configuration.RecurseFlag_"/run/noload/nodelete/nodisplay"
+  set flags = "/"_configuration.RecurseFlag_"/run/load/nodelete/nodisplay"
   set testSpecs = configuration.TestSuite_":"_configuration.TestCase
 }
 
@@ -33,7 +34,7 @@ ClassMethod DescribeConfiguration(testSpecs As %String, flags As %String) [ Priv
   write !, "Runner: Notice: The test execution summary will be displayed at the end."
 }
 
-ClassMethod RunSilently(testSpecs As %String, flags As %String, ByRef params As %String) As %Status [ Private ]
+ClassMethod RunSilently(testManager As %String = "%UnitTest.Manager", testSpecs As %String, flags As %String, ByRef params As %String) As %Status [ Private ]
 {
   set io = $io
 
@@ -42,7 +43,8 @@ ClassMethod RunSilently(testSpecs As %String, flags As %String, ByRef params As
 
   if testSpecs = ":" set testSpecs = ""
 
-  set sc = ##class(%UnitTest.Manager).RunTest(testSpecs, flags, .params)
+  set testManagerClass = $select(testManager '= "" : testManager, 1: "%UnitTest.Manager")
+  set sc = $classmethod(testManagerClass, "RunTest", testSpecs, flags, .params)
 
   use io
   close $$$NULL
@@ -70,47 +72,45 @@ ClassMethod HasAnyTestBeenExecuted() As %Boolean [ Private ]
 
 ClassMethod HandleUnitTestFeedback(configuration As CI.Configuration) As %Status [ Private ]
 {
-  new $namespace
-
-  set environmentNS = configuration.GetEnv("CI_NAMESPACE")
-  set $namespace = $get(environmentNS, $namespace)
-
   if '..HasAnyTestBeenExecuted() return $$$OK
-  do ..GetAssertionResult(.failure, .success)
+  do ..GetAssertionResult(.failure, .success, .error)
 
-  do ..WriteAssertionResult(failure, success)
+  do ..WriteAssertionResult(failure, success, error)
   do ..WriteSummaryFromFailures()
-  do ..WriteEllapsedTime()
+  do ..WriteElapsedTime()
 
-  $$$QuitOnError(..WriteAssertionStatistics(failure, success))
+  $$$QuitOnError(..WriteAssertionStatistics(failure, success, error))
   return $$$OK
 }
 
-ClassMethod WriteEllapsedTime() [ Private ]
+ClassMethod WriteElapsedTime() [ Private ]
 {
   set time = 0
   &sql(SELECT TOP 1 DURATION INTO :time FROM %UNITTEST_RESULT.TESTINSTANCE)
   write $$$BOLDDEFAULT("Elapsed time: "_time_" seconds."), !!
 }
 
-ClassMethod GetAssertionResult(Output failure = 0, Output success = 0) [ Private ]
+ClassMethod GetAssertionResult(Output failure As %Integer = 0, Output success As %Integer = 0, Output error As %Integer = 0) [ Private ]
 {
-  &sql(SELECT COUNT (ID) INTO :success FROM %UNITTEST_RESULT.TESTASSERT WHERE STATUS = 1)
-  &sql(SELECT COUNT (ID) INTO :failure FROM %UNITTEST_RESULT.TESTASSERT WHERE STATUS = 0)
+  &sql(SELECT COUNT (ID) INTO :success FROM %UNITTEST_RESULT.TESTASSERT WHERE STATUS = 1 AND ACTION <> 'LogMessage')
+  &sql(SELECT COUNT (ID) INTO :failure FROM %UNITTEST_RESULT.TESTASSERT WHERE STATUS = 0 AND ACTION <> 'LogMessage')
+  &sql(SELECT COUNT (ID) INTO :error FROM %UNITTEST_RESULT.TESTMETHOD WHERE ERRORACTION IS NOT NULL AND ERRORACTION <> '')
 }
 
-ClassMethod WriteAssertionResult(failure As %Integer, success As %Integer) [ Private ]
+ClassMethod WriteAssertionResult(failure As %Integer, success As %Integer, error As %Integer) [ Private ]
 {
+  if error > 0 write $$$BOLDRED("One or more tests have ERRORED."), ! quit
   if failure > 0 write $$$BOLDRED("One or more tests have FAILED."), !  quit
   if success > 0 write $$$BOLDGREEN("All tests have PASSED."), !  quit
 }
 
-ClassMethod WriteAssertionStatistics(failure As %Integer = 0, success As %Integer = 0) As %Status [ Private ]
+ClassMethod WriteAssertionStatistics(failure As %Integer = 0, success As %Integer = 0, error As %Integer = 0) As %Status [ Private ]
 {
   write $$$BOLDGREEN("✓ "_success_ " assertions have passed."), !
   write $$$BOLDRED("✗ "_failure_" assertions have failed."), !
+  if error > 0 write $$$BOLDRED("! "_error_" assertions have errored."), !
 
-  IF failure > 0 return $$$ERROR($$$GeneralError, "ASSERTION ERROR.")
+  IF failure > 0 || (error > 0) return $$$ERROR($$$GeneralError, "ASSERTION ERROR.")
   return $$$OK
 }
 
@@ -122,7 +122,7 @@ ClassMethod  WriteSummaryFromFailures() [ Private ]
   for i=1:1:testResult.TestSuites.Count() {
     set testSuite = testResult.TestSuites.GetAt(i)
     if testSuite.Status = 0 {
-      write $$$RED("Assertion failure on test suite located at "_testSuite.Name), !
+      write $$$RED("Assertion failure on test suite from "_testSuite.Name), !
       for j=1:1:testSuite.TestCases.Count() {
         set testCase = testSuite.TestCases.GetAt(j)
         if testCase.Status = 0 {