Neg tests check files for // error markers (rebased and updated)

src/dotty/tools/dotc/reporting/Reporter.scala

@@ -223,6 +223,8 @@ abstract class Reporter {
   var warningCount = 0
   def hasErrors = errorCount > 0
   def hasWarnings = warningCount > 0
+  private var errors: List[Error] = Nil
+  def allErrors = errors
 
   /** Have errors been reported by this reporter, or in the
    *  case where this is a StoreReporter, by an outer reporter?
@@ -238,7 +240,9 @@ abstract class Reporter {
       d match {
         case d: ConditionalWarning if !d.enablingOption.value => unreportedWarnings(d.enablingOption.name) += 1
         case d: Warning => warningCount += 1
-        case d: Error => errorCount += 1
+        case d: Error =>
+          errors = d :: errors
+          errorCount += 1
         case d: Info => // nothing to do here
         // match error if d is something else
       }

test/test/CompilerTest.scala

@@ -3,9 +3,12 @@ package test
 import dotty.partest.DPConfig
 import dotty.tools.dotc.{Main, Bench, Driver}
 import dotty.tools.dotc.reporting.Reporter
+import dotty.tools.dotc.util.SourcePosition
+import dotty.tools.dotc.config.CompilerCommand
 import scala.collection.mutable.ListBuffer
-import scala.reflect.io.{ Path, Directory, File => SFile }
+import scala.reflect.io.{ Path, Directory, File => SFile, AbstractFile }
 import scala.tools.partest.nest.{ FileManager, NestUI }
+import scala.annotation.tailrec
 import java.io.{ RandomAccessFile, File => JFile }
 
 import org.junit.Test
@@ -178,13 +181,140 @@ abstract class CompilerTest {
 
   // ========== HELPERS =============
 
-  private def compileArgs(args: Array[String], xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = {
+  private def compileArgs(args: Array[String], xerrors: Int = 0)
+      (implicit defaultOptions: List[String]): Unit = {
     val allArgs = args ++ defaultOptions
     val processor = if (allArgs.exists(_.startsWith("#"))) Bench else Main
-    val nerrors = processor.process(allArgs).errorCount
+    val reporter = processor.process(allArgs)
+
+    val nerrors = reporter.errorCount
     assert(nerrors == xerrors, s"Wrong # of errors. Expected: $xerrors, found: $nerrors")
+
+    // NEG TEST
+    if (xerrors > 0) {
+      val errorLines = reporter.allErrors.map(_.pos)
+      // reporter didn't record as many errors as its errorCount says
+      assert(errorLines.length == nerrors, s"Not enough errors recorded.")
+
+      val allFiles = (allArgs filter {
+          arg => !arg.startsWith("-") && (arg.endsWith(".scala") || arg.endsWith(".java"))
+      }).toList
+      val expectedErrorsPerFile = allFiles.map(getErrors(_))
+
+      // Some compiler errors have an associated source position. Each error
+      // needs to correspond to a "// error" marker on that line in the source
+      // file and vice versa.
+      // Other compiler errors don't have an associated source position. Their
+      // number should correspond to the total count of "// nopos-error"
+      // markers in all files
+      val (errorsByFile, errorsWithoutPos) = errorLines.groupBy(_.source.file).toList.partition(_._1.toString != "<no source>")
+
+      // check errors with source position
+      val foundErrorsPerFile = errorsByFile.map({ case (fileName, errorList) =>
+        val posErrorLinesToNr = errorList.groupBy(_.line).toList.map({ case (line, list) => (line, list.length) }).sortBy(_._1)
+        ErrorsInFile(fileName.toString, 0, posErrorLinesToNr)
+      })
+      val expectedErrorsPerFileZeroed = expectedErrorsPerFile.map({
+        case ErrorsInFile(fileName, _, posErrorLinesToNr) =>
+          ErrorsInFile(fileName.toString, 0, posErrorLinesToNr)
+      })
+      checkErrorsWithPosition(expectedErrorsPerFileZeroed, foundErrorsPerFile)
+
+      // check errors without source position
+      val expectedNoPos = expectedErrorsPerFile.map(_.noposErrorNr).sum
+      val foundNoPos = errorsWithoutPos.map(_._2.length).sum
+      assert(foundNoPos == expectedNoPos,
+        s"Wrong # of errors without source position. Expected (all files): $expectedNoPos, found (compiler): $foundNoPos")
+    }
   }
 
+  // ========== NEG TEST HELPERS =============
+
+  /** Captures the number of nopos-errors in the given file and the number of
+    * errors with a position, represented as a tuple of source line and number
+    * of errors on that line. */
+  case class ErrorsInFile(fileName: String, noposErrorNr: Int, posErrorLinesToNr: List[(Int, Int)])
+
+  /** Extracts the errors expected for the given neg test file. */
+  def getErrors(fileName: String): ErrorsInFile = {
+    val content = SFile(fileName).slurp
+    val (line, rest) = content.span(_ != '\n')
+
+    @tailrec
+    def checkLine(line: String, rest: String, index: Int, noposAcc: Int, posAcc: List[(Int, Int)]): ErrorsInFile = {
+      val posErrors = "// ?error".r.findAllIn(line).length
+      val newPosAcc = if (posErrors > 0) (index, posErrors) :: posAcc else posAcc
+      val newNoPosAcc = noposAcc + "// ?nopos-error".r.findAllIn(line).length
+      val (newLine, newRest) = rest.span(_ != '\n')
+      if (newRest.isEmpty)
+        ErrorsInFile(fileName.toString, newNoPosAcc, newPosAcc.reverse)
+      else
+        checkLine(newLine, newRest.tail, index + 1, newNoPosAcc, newPosAcc) // skip leading '\n'
+    }
+
+    checkLine(line, rest.tail, 0, 0, Nil) // skip leading '\n'
+  }
+
+  /** Asserts that the expected and found number of errors correspond, and
+    * otherwise throws an error with the filename, plus optionally a line
+    * number if available. */
+  def errorMsg(fileName: String, lineNumber: Option[Int], exp: Int, found: Int) = {
+    val i = lineNumber.map({ i => ":" + (i + 1) }).getOrElse("")
+    assert(found == exp, s"Wrong # of errors for $fileName$i. Expected (file): $exp, found (compiler): $found")
+  }
+
+  /** Compares the expected with the found errors and creates a nice error
+    * message if they don't agree. */
+  def checkErrorsWithPosition(expected: List[ErrorsInFile], found: List[ErrorsInFile]): Unit = {
+      // create nice error messages
+      expected.diff(found) match {
+        case Nil => // nothing missing
+        case ErrorsInFile(fileName, _, expectedLines) :: xs =>
+          found.find(_.fileName == fileName) match {
+            case None =>
+              // expected some errors, but none found for this file
+              errorMsg(fileName, None, expectedLines.map(_._2).sum, 0)
+            case Some(ErrorsInFile(_,_,foundLines)) =>
+              // found wrong number/location of markers for this file
+              compareLines(fileName, expectedLines, foundLines)
+          }
+      }
+
+      found.diff(expected) match {
+        case Nil => // nothing missing
+        case ErrorsInFile(fileName, _, foundLines) :: xs =>
+          expected.find(_.fileName == fileName) match {
+            case None =>
+              // found some errors, but none expected for this file
+              errorMsg(fileName, None, 0, foundLines.map(_._2).sum)
+            case Some(ErrorsInFile(_,_,expectedLines)) =>
+              // found wrong number/location of markers for this file
+              compareLines(fileName, expectedLines, foundLines)
+          }
+      }
+  }
+
+  /** Gives an error message for one line where the expected number of errors and
+    * the number of compiler errors differ. */
+  def compareLines(fileName: String, expectedLines: List[(Int, Int)], foundLines: List[(Int, Int)]) = {
+    expectedLines.foreach({ case (line, expNr) =>
+      foundLines.find(_._1 == line) match {
+        case Some((_, `expNr`)) => // this line is ok
+        case Some((_, foundNr)) => errorMsg(fileName, Some(line), expNr, foundNr)
+        case None               => errorMsg(fileName, Some(line), expNr, 0)
+      }
+    })
+    foundLines.foreach({ case (line, foundNr) =>
+      expectedLines.find(_._1 == line) match {
+        case Some((_, `foundNr`)) => // this line is ok
+        case Some((_, expNr))     => errorMsg(fileName, Some(line), expNr, foundNr)
+        case None                 => errorMsg(fileName, Some(line), 0,     foundNr)
+      }
+    })
+  }
+
+  // ========== PARTEST HELPERS =============
+
   // In particular, don't copy flags from scalac tests
   private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java")
 

tests/neg/abstract-override.scala

@@ -1,6 +1,6 @@
 trait T { def foo: Int }
-trait T1 extends T { override def foo = super.foo }
-trait T2 extends T { override def foo = super.foo }
+trait T1 extends T { override def foo = super.foo } // error: method foo in trait T is accessed from super.
+trait T2 extends T { override def foo = super.foo } // error: method foo in trait T is accessed from super.
 object Test extends T2 with T1 {
   def main(args: Array[String]) = {
     assert(foo == 3)

tests/neg/amp.scala

@@ -2,14 +2,14 @@ object Test extends dotty.runtime.LegacyApp {
 
   def foo() = {
     def f: Int = 1
-    val x = f _
+    val x = f _ // error: not a function: => Int(f)
     x
   }
 
   def bar(g: => Int) = {
-    g _
+    g _ // error: not a function: => Int(g)
   }
 
-  Console.println((bar{ Console.println("g called"); 42 })())
-  Console.println(foo()())
+  Console.println((bar{ Console.println("g called"); 42 })()) // error: method bar in object Test$ does not take more parameters
+  Console.println(foo()()) // error: method foo in object Test$ does not take more parameters
 }

tests/neg/arrayclone-new.scala

@@ -7,7 +7,7 @@ object Test extends dotty.runtime.LegacyApp{
 }
 
 object ObjectArrayClone{
-  val it : Array[String] = Array("1", "0");
+  val it : Array[String] = Array("1", "0");           // error
   val cloned = it.clone();
   assert(cloned.sameElements(it));
   cloned(0) = "0";
@@ -22,7 +22,7 @@ object PolymorphicArrayClone{
     assert(it(0) == one)
   }
 
-  testIt(Array("one", "two"), "one", "two");
+  testIt(Array("one", "two"), "one", "two");          // error
 
   class Mangler[T: ClassTag](ts : T*){
     // this will always be a BoxedAnyArray even after we've unboxed its contents.

tests/neg/assignments.scala

@@ -13,11 +13,11 @@ object assignments {
     x = x + 1
     x *= 2
 
-    x_= = 2  // should give missing arguments + reassignment to val
+    x_= = 2  // error should give missing arguments + // error reassignment to val
   }
 
   var c = new C
-  import c._ // should give: prefix is not stable
+  import c._ // error should give: prefix is not stable
   x = x + 1
   x *= 2
 }

tests/neg/autoTuplingTest.scala

@@ -1,11 +1,11 @@
 import dotty.language.noAutoTupling
 
-object autoTuplingNeg {
+object autoTuplingNeg2 {
 
-  val x = Some(1, 2)
+  val x = Some(1, 2) // error: too many arguments for method apply: (x: A)Some[A]
 
   x match {
-    case Some(a, b) => a + b
+    case Some(a, b) => a + b // error: wrong number of argument patterns for Some // error: not found: b
     case None =>
   }
 }

tests/neg/blockescapesNeg.scala

@@ -1,6 +1,6 @@
 object blockescapesNeg {
   def m0 = { object Foo { class Bar { val field = 2 }} ; new Foo.Bar }
-  m0.field
+  m0.field                                            // error
   class A[T]
   def m1 = { val x = 1; new A[x.type]}
 }

tests/neg/bounds.scala

@@ -1,10 +1,10 @@
 object Test {
   def g[B >: String <: Int](x: B): Int = x
   def main(args: Array[String]): Unit = {
-    g("foo")
+    g("foo") // error: Type argument String' does not conform to upper bound Int
   }
   def baz[X >: Y, Y <: String](x: X, y: Y) = (x, y)
 
-  baz[Int, String](1, "abc")
+  baz[Int, String](1, "abc") // error: Type argument Int does not conform to lower bound Y
 
 }

tests/neg/boundspropagation.scala

@@ -13,15 +13,15 @@ object test2 {
 
 
   def f(x: Any): Tree[Null] = x match {
-    case y: Tree[_] => y
+    case y: Tree[_] => y                              // error
   }
 }
 object test3 {
   class Tree[+T >: Null]
 
 
   def f(x: Any): Tree[Null] = x match {
-    case y: Tree[_] => y
+    case y: Tree[_] => y                              // error
   }
 }
 
@@ -34,11 +34,11 @@ object test4 {
     class Tree[-S, -T >: Option[S]]
 
     def g(x: Any): Tree[_, _ <: Option[N]] = x match {
-      case y: Tree[_, _] => y
+      case y: Tree[_, _] => y                         // error
     }
   }
 }
 
 class Test5 {
-"": ({ type U = this.type })#U
+"": ({ type U = this.type })#U                        // error // error
 }

tests/neg/companions.scala

@@ -8,7 +8,7 @@ object companionsNeg {
 
     { object C {
       private val p = 1
-      println(new C().q)
+      println(new C().q)   // error
     }}
   }
 

tests/neg/cycles.scala

@@ -22,8 +22,8 @@ class C {
 
 class E {
   class F {
-    type T <: x.type // error: not stable
-    val z: x.type = ??? // error: not stable
+    type T <: x.type // old-error: not stable
+    val z: x.type = ??? // old-error: not stable
   }
   lazy val x: F#T = ???
 }
@@ -37,6 +37,6 @@ class T2 {
   type U = X | Int
 }
 object T12 {
-  ??? : (T1 {})#U // error: conflicting bounds
-  ??? : (T2 {})#U // error: conflicting bounds
+  ??? : (T1 {})#U // old-error: conflicting bounds
+  ??? : (T2 {})#U // old-error: conflicting bounds
 }

tests/neg/escapingRefs.scala

@@ -3,8 +3,8 @@ object O {
   class B
   def f[T](x: T, y: T): T = y
 
-  val x: A = f(new A { }, new B { })
+  val x: A = f(new A { }, new B { })                  // error
 
   val y = f({ class C { def member: Int = 1 }; new C }, { class C { def member: Int = 1 }; new C })
-  val z = y.member
+  val z = y.member                                    // error
 }

tests/neg/final-sealed.scala

@@ -1,4 +1,4 @@
 final class A
-class B extends A
-class C extends Option[Int]
+class B extends A // error: cannot extend final class A
+class C extends Option[Int] // error: cannot extend sealed class Option in different compilation unit
 

tests/neg/firstError.scala

@@ -1,4 +1,4 @@
-.
+. // error: expected class or object definition
 
-\u890u3084eu
+\u890u3084eu // error: error in unicode escape // error: illegal character '\uffff'
 

tests/neg/i0091-infpaths.scala

@@ -2,15 +2,15 @@ object infpaths {
 
   object a {
     trait T { t =>
-      type M <: t.b.M
+      type M <: t.b.M                                 // error
       type T <: a.T
       val b: t.T
     }
     val x: a.T = ???
   }
 
   val m1: a.x.M = ???
-  val m2: a.x.b.M = m1
-  val m3: a.x.b.b.M = m2
+  val m2: a.x.b.M = m1                                // error
+  val m3: a.x.b.b.M = m2                              // error
 
 }

tests/neg/i0248-inherit-refined.scala

@@ -1,10 +1,10 @@
 object test {
   class A { type T }
   type X = A { type T = Int }
-  class B extends X
+  class B extends X                                   // error
   type Y = A & B
-  class C extends Y
+  class C extends Y                                   // error
   type Z = A | B
-  class D extends Z
-  abstract class E extends ({ val x: Int })
+  class D extends Z                                   // error
+  abstract class E extends ({ val x: Int })           // error
 }