Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Neg tests: remove xerror parameter from neg tests (#1110) #1122

Merged
merged 3 commits into from
Mar 3, 2016
Merged

Neg tests: remove xerror parameter from neg tests (#1110) #1122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
447256c
Neg tests: remove xerror parameter from tests (compute based on // er…
VladimirNik Feb 22, 2016
92b6677
Neg tests: remove negTest flag; neg tests are binded to neg directory
VladimirNik Feb 23, 2016
2b8cd5a
Neg tests: change val computeFilePathsAndExpErrors to def
VladimirNik Mar 3, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 14 additions & 79 deletions test/dotc/tests.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,87 +108,22 @@ class tests extends CompilerTest {

@Test def new_all = compileFiles(newDir, twice)

@Test def neg_abstractOverride() = compileFile(negDir, "abstract-override", xerrors = 2)
@Test def neg_blockescapes() = compileFile(negDir, "blockescapesNeg", xerrors = 1)
@Test def neg_bounds() = compileFile(negDir, "bounds", xerrors = 2)
@Test def neg_functionArity() = compileFile(negDir, "function-arity", xerrors = 7)
@Test def neg_typedapply() = compileFile(negDir, "typedapply", xerrors = 3)
@Test def neg_typedIdents() = compileDir(negDir, "typedIdents", xerrors = 2)
@Test def neg_assignments() = compileFile(negDir, "assignments", xerrors = 3)
@Test def neg_typers() = compileFile(negDir, "typers", xerrors = 14)(allowDoubleBindings)
@Test def neg_privates() = compileFile(negDir, "privates", xerrors = 2)
@Test def neg_rootImports = compileFile(negDir, "rootImplicits", xerrors = 2)
@Test def neg_templateParents() = compileFile(negDir, "templateParents", xerrors = 3)
@Test def neg_autoTupling = compileFile(posDir, "autoTuplingTest", args = "-language:noAutoTupling" :: Nil, xerrors = 3)
@Test def neg_autoTupling2 = compileFile(negDir, "autoTuplingTest", xerrors = 3)
@Test def neg_companions = compileFile(negDir, "companions", xerrors = 1)
@Test def namedParams = compileFile(negDir, "named-params", xerrors = 14)
@Test def neg_over = compileFile(negDir, "over", xerrors = 3)
@Test def neg_overrides = compileFile(negDir, "overrides", xerrors = 14)
@Test def neg_overrideClass = compileFile(negDir, "overrideClass", List("-language:Scala2"), xerrors = 1)
@Test def neg_i39 = compileFile(negDir, "i39", xerrors = 2)
@Test def neg_i50_volatile = compileFile(negDir, "i50-volatile", xerrors = 3)
@Test def neg_zoo = compileFile(negDir, "zoo", xerrors = 12)
@Test def neg_all = compileFiles(negDir, verbose = true, compileSubDirs = false)
@Test def neg_typedIdents() = compileDir(negDir, "typedIdents")

val negCustomArgs = negDir + "customArgs/"
@Test def neg_typers() = compileFile(negCustomArgs, "typers")(allowDoubleBindings)
@Test def neg_overrideClass = compileFile(negCustomArgs, "overrideClass", List("-language:Scala2"))
@Test def neg_autoTupling = compileFile(negCustomArgs, "autoTuplingTest", args = "-language:noAutoTupling" :: Nil)
@Test def neg_i1050 = compileFile(negCustomArgs, "i1050", List("-strict"))

val negTailcallDir = negDir + "tailcall/"
@Test def neg_tailcall_t1672b = compileFile(negTailcallDir, "t1672b", xerrors = 5)
@Test def neg_tailcall_t3275 = compileFile(negTailcallDir, "t3275", xerrors = 1)
@Test def neg_tailcall_t6574 = compileFile(negTailcallDir, "t6574", xerrors = 2)
@Test def neg_tailcall = compileFile(negTailcallDir, "tailrec", xerrors = 7)
@Test def neg_tailcall2 = compileFile(negTailcallDir, "tailrec-2", xerrors = 2)
@Test def neg_tailcall3 = compileFile(negTailcallDir, "tailrec-3", xerrors = 2)

@Test def neg_t1843_variances = compileFile(negDir, "t1843-variances", xerrors = 1)
@Test def neg_t2660_ambi = compileFile(negDir, "t2660", xerrors = 2)
@Test def neg_t2994 = compileFile(negDir, "t2994", xerrors = 2)
@Test def neg_subtyping = compileFile(negDir, "subtyping", xerrors = 5)
@Test def neg_variances = compileFile(negDir, "variances", xerrors = 2)
@Test def neg_variancesConstr = compileFile(negDir, "variances-constr", xerrors = 2)
@Test def neg_i871_missingReturnType = compileFile(negDir, "i871", xerrors = 2)
@Test def neg_badAuxConstr = compileFile(negDir, "badAuxConstr", xerrors = 2)
@Test def neg_typetest = compileFile(negDir, "typetest", xerrors = 1)
@Test def neg_t1569_failedAvoid = compileFile(negDir, "t1569-failedAvoid", xerrors = 1)
@Test def neg_clashes = compileFile(negDir, "clashes", xerrors = 2)
@Test def neg_cycles = compileFile(negDir, "cycles", xerrors = 7)
@Test def neg_boundspropagation = compileFile(negDir, "boundspropagation", xerrors = 5)
@Test def neg_refinedSubtyping = compileFile(negDir, "refinedSubtyping", xerrors = 2)
@Test def neg_hklower = compileFile(negDir, "hklower", xerrors = 4)
@Test def neg_Iter2 = compileFile(negDir, "Iter2", xerrors = 2)
@Test def neg_i0091_infpaths = compileFile(negDir, "i0091-infpaths", xerrors = 3)
@Test def neg_i0248_inherit_refined = compileFile(negDir, "i0248-inherit-refined", xerrors = 4)
@Test def neg_i0281 = compileFile(negDir, "i0281-null-primitive-conforms", xerrors = 3)
@Test def neg_i324 = compileFile(negDir, "i324", xerrors = 2)
@Test def neg_i583 = compileFile(negDir, "i0583-skolemize", xerrors = 2)
@Test def neg_i941 = compileFile(negDir, "i941", xerrors = 3)
@Test def neg_finalSealed = compileFile(negDir, "final-sealed", xerrors = 2)
@Test def neg_i645 = compileFile(negDir, "amp", xerrors = 4)
@Test def neg_i705 = compileFile(negDir, "i705-inner-value-class", xerrors = 7)
@Test def neg_i803 = compileFile(negDir, "i803", xerrors = 2)
@Test def neg_i866 = compileFile(negDir, "i866", xerrors = 2)
@Test def neg_i974 = compileFile(negDir, "i974", xerrors = 2)
@Test def neg_i997 = compileFile(negDir, "i997", xerrors = 15)
@Test def neg_i997a = compileFile(negDir, "i997a", xerrors = 2)
@Test def neg_i1050 = compileFile(negDir, "i1050", List("-strict"), xerrors = 11)
@Test def neg_i1050a = compileFile(negDir, "i1050a", xerrors = 2)
@Test def neg_i1050c = compileFile(negDir, "i1050c", xerrors = 8)
@Test def neg_moduleSubtyping = compileFile(negDir, "moduleSubtyping", xerrors = 4)
@Test def neg_escapingRefs = compileFile(negDir, "escapingRefs", xerrors = 2)
@Test def neg_instantiateAbstract = compileFile(negDir, "instantiateAbstract", xerrors = 8)
@Test def neg_partialApplications = compileFile(negDir, "partialApplications", xerrors = 3)
@Test def neg_selfInheritance = compileFile(negDir, "selfInheritance", xerrors = 6)
@Test def neg_selfreq = compileFile(negDir, "selfreq", xerrors = 2)
@Test def neg_singletons = compileFile(negDir, "singletons", xerrors = 8)
@Test def neg_ski = compileFile(negDir, "ski", xerrors = 12)
@Test def neg_traitParamsTyper = compileFile(negDir, "traitParamsTyper", xerrors = 5)
@Test def neg_traitParamsMixin = compileFile(negDir, "traitParamsMixin", xerrors = 2)
@Test def neg_firstError = compileFile(negDir, "firstError", xerrors = 3)
@Test def neg_implicitLowerBound = compileFile(negDir, "implicit-lower-bound", xerrors = 1)
@Test def neg_validate = compileFile(negDir, "validate", xerrors = 18)
@Test def neg_validateParsing = compileFile(negDir, "validate-parsing", xerrors = 7)
@Test def neg_validateRefchecks = compileFile(negDir, "validate-refchecks", xerrors = 2)
@Test def neg_skolemize = compileFile(negDir, "skolemize", xerrors = 2)
@Test def neg_nested_bounds = compileFile(negDir, "nested_bounds", xerrors = 1)
@Test def neg_valueClasses = compileFile(negDir, "valueClasses", xerrors = 2)
@Test def neg_tailcall_t1672b = compileFile(negTailcallDir, "t1672b")
@Test def neg_tailcall_t3275 = compileFile(negTailcallDir, "t3275")
@Test def neg_tailcall_t6574 = compileFile(negTailcallDir, "t6574")
@Test def neg_tailcall = compileFile(negTailcallDir, "tailrec")
@Test def neg_tailcall2 = compileFile(negTailcallDir, "tailrec-2")
@Test def neg_tailcall3 = compileFile(negTailcallDir, "tailrec-3")

@Test def run_all = runFiles(runDir)

Expand Down
110 changes: 64 additions & 46 deletions test/test/CompilerTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ abstract class CompilerTest {
val defaultOutputDir: String

/** Override to filter out tests that should not be run by partest. */
def partestableFile(prefix: String, fileName: String, extension: String, args: List[String], xerrors: Int) = true
def partestableDir(prefix: String, dirName: String, args: List[String], xerrors: Int) = true
def partestableList(testName: String, files: List[String], args: List[String], xerrors: Int) = true
def partestableFile(prefix: String, fileName: String, extension: String, args: List[String]) = true
def partestableDir(prefix: String, dirName: String, args: List[String]) = true
def partestableList(testName: String, files: List[String], args: List[String]) = true

val generatePartestFiles = {
/* Because we fork in test, the JVM in which this JUnit test runs has a
Expand All @@ -68,70 +68,80 @@ abstract class CompilerTest {
val logFile = if (!generatePartestFiles) None else Some(CompilerTest.init)

/** Always run with JUnit. */
def compileLine(cmdLine: String, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = {
def compileLine(cmdLine: String)(implicit defaultOptions: List[String]): Unit = {
if (generatePartestFiles)
log("WARNING: compileLine will always run with JUnit, no partest files generated.")
compileArgs(cmdLine.split("\n"), xerrors)
compileArgs(cmdLine.split("\n"), Nil)
}

/** Compiles the given code file.
*
* @param prefix the parent directory (including separator at the end)
* @param fileName the filename, by default without extension
* @param args arguments to the compiler
* @param xerrors if > 0, this test is a neg test with the expected number
* of compiler errors. Otherwise, this is a pos test.
* @param extension the file extension, .scala by default
* @param defaultOptions more arguments to the compiler
*/
def compileFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0,
extension: String = ".scala", runTest: Boolean = false)
def compileFile(prefix: String, fileName: String, args: List[String] = Nil, extension: String = ".scala", runTest: Boolean = false)
(implicit defaultOptions: List[String]): Unit = {
if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions, xerrors)) {
val filePath = s"$prefix$fileName$extension"
val expErrors = expectedErrors(filePath)
if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions)) {
if (runTest)
log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$fileName$extension")
compileArgs((s"$prefix$fileName$extension" :: args).toArray, xerrors)
compileArgs((s"$filePath" :: args).toArray, expErrors)
} else {
val kind = testKind(prefix, xerrors, runTest)
val kind = testKind(prefix, runTest)
log(s"generating partest files for test file: $prefix$fileName$extension of kind $kind")

val sourceFile = new JFile(prefix + fileName + extension)
if (sourceFile.exists) {
val firstDest = SFile(DPConfig.testRoot + JFile.separator + kind + JFile.separator + fileName + extension)
val xerrors = expErrors.map(_.totalErrors).sum
computeDestAndCopyFiles(sourceFile, firstDest, kind, args ++ defaultOptions, xerrors.toString)
} else {
throw new java.io.FileNotFoundException(s"Unable to locate test file $prefix$fileName")
}
}
}
def runFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0,
extension: String = ".scala")(implicit defaultOptions: List[String]): Unit =
compileFile(prefix, fileName, args, xerrors, extension, true)
def runFile(prefix: String, fileName: String, args: List[String] = Nil, extension: String = ".scala")
(implicit defaultOptions: List[String]): Unit = {
compileFile(prefix, fileName, args, extension, true)
}

/** Compiles the code files in the given directory together. If args starts
* with "-deep", all files in subdirectories (and so on) are included. */
def compileDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0, runTest: Boolean = false)
def compileDir(prefix: String, dirName: String, args: List[String] = Nil, runTest: Boolean = false)
(implicit defaultOptions: List[String]): Unit = {
if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions, xerrors)) {
if (runTest)
log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName")
def computeFilePathsAndExpErrors = {
val dir = Directory(prefix + dirName)
val (files, normArgs) = args match {
case "-deep" :: args1 => (dir.deepFiles, args1)
case _ => (dir.files, args)
}
val fileNames = files.toArray.map(_.toString).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
compileArgs(fileNames ++ normArgs, xerrors)
val filePaths = files.toArray.map(_.toString).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
val expErrors = expectedErrors(filePaths.toList)
(filePaths, normArgs, expErrors)
}
if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions)) {
if (runTest)
log(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName")
val (filePaths, normArgs, expErrors) = computeFilePathsAndExpErrors
compileArgs(filePaths ++ normArgs, expErrors)
} else {
val (sourceDir, flags, deep) = args match {
case "-deep" :: args1 => (flattenDir(prefix, dirName), args1 ++ defaultOptions, "deep")
case _ => (new JFile(prefix + dirName), args ++ defaultOptions, "shallow")
}
val kind = testKind(prefix, xerrors, runTest)
val kind = testKind(prefix, runTest)
log(s"generating partest files for test directory ($deep): $prefix$dirName of kind $kind")

if (sourceDir.exists) {
val firstDest = Directory(DPConfig.testRoot + JFile.separator + kind + JFile.separator + dirName)
val xerrors = if (isNegTest(prefix)) {
val (_, _, expErrors) = computeFilePathsAndExpErrors
expErrors.map(_.totalErrors).sum
} else 0
computeDestAndCopyFiles(sourceDir, firstDest, kind, flags, xerrors.toString)
if (deep == "deep")
Directory(sourceDir).deleteRecursively
Expand All @@ -140,67 +150,73 @@ abstract class CompilerTest {
}
}
}
def runDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0)
def runDir(prefix: String, dirName: String, args: List[String] = Nil)
(implicit defaultOptions: List[String]): Unit =
compileDir(prefix, dirName, args, xerrors, true)
compileDir(prefix, dirName, args, true)

/** Compiles each source in the directory path separately by calling
* compileFile resp. compileDir. */
def compileFiles(path: String, args: List[String] = Nil, verbose: Boolean = true, runTest: Boolean = false)
(implicit defaultOptions: List[String]): Unit = {
def compileFiles(path: String, args: List[String] = Nil, verbose: Boolean = true, runTest: Boolean = false,
compileSubDirs: Boolean = true)(implicit defaultOptions: List[String]): Unit = {
val dir = Directory(path)
val fileNames = dir.files.toArray.map(_.jfile.getName).filter(name => (name endsWith ".scala") || (name endsWith ".java"))
for (name <- fileNames) {
if (verbose) log(s"testing $path$name")
compileFile(path, name, args, 0, "", runTest)
}
for (subdir <- dir.dirs) {
if (verbose) log(s"testing $subdir")
compileDir(path, subdir.jfile.getName, args, 0, runTest)
compileFile(path, name, args, "", runTest)
}
if (compileSubDirs)
for (subdir <- dir.dirs) {
if (verbose) log(s"testing $subdir")
compileDir(path, subdir.jfile.getName, args, runTest)
}
}
def runFiles(path: String, args: List[String] = Nil, verbose: Boolean = true)
(implicit defaultOptions: List[String]): Unit =
compileFiles(path, args, verbose, true)

/** Compiles the given list of code files. */
def compileList(testName: String, files: List[String], args: List[String] = Nil, xerrors: Int = 0)
def compileList(testName: String, files: List[String], args: List[String] = Nil)
(implicit defaultOptions: List[String]): Unit = {
if (!generatePartestFiles || !partestableList(testName, files, args ++ defaultOptions, xerrors)) {
compileArgs((files ++ args).toArray, xerrors)
if (!generatePartestFiles || !partestableList(testName, files, args ++ defaultOptions)) {
val expErrors = expectedErrors(files)
compileArgs((files ++ args).toArray, expErrors)
} else {
val destDir = Directory(DPConfig.testRoot + JFile.separator + testName)
files.foreach({ file =>
val jfile = new JFile(file)
recCopyFiles(jfile, destDir / jfile.getName)
})
compileDir(DPConfig.testRoot + JFile.separator, testName, args, xerrors)
compileDir(DPConfig.testRoot + JFile.separator, testName, args)
destDir.deleteRecursively
}
}

// ========== HELPERS =============

private def compileArgs(args: Array[String], xerrors: Int = 0)
private def expectedErrors(filePaths: List[String]): List[ErrorsInFile] = if (filePaths.exists(isNegTest(_))) filePaths.map(getErrors(_)) else Nil

private def expectedErrors(filePath: String): List[ErrorsInFile] = expectedErrors(List(filePath))

private def isNegTest(testPath: String) = testPath.contains(JFile.separator + "neg" + JFile.separator)

private def compileArgs(args: Array[String], expectedErrorsPerFile: List[ErrorsInFile])
(implicit defaultOptions: List[String]): Unit = {
val allArgs = args ++ defaultOptions
val processor = if (allArgs.exists(_.startsWith("#"))) Bench else Main
val reporter = processor.process(allArgs)

val nerrors = reporter.errorCount
assert(nerrors == xerrors, s"Wrong # of errors. Expected: $xerrors, found: $nerrors")

val xerrors = (expectedErrorsPerFile map {_.totalErrors}).sum
assert(nerrors == xerrors,
s"""Wrong # of errors. Expected: $xerrors, found: $nerrors
|Files with expected errors: ${expectedErrorsPerFile.collect{ case er if er.totalErrors > 0 => er.fileName} }
""".stripMargin)
// NEG TEST
if (xerrors > 0) {
val errorLines = reporter.allErrors.map(_.pos)
// reporter didn't record as many errors as its errorCount says
assert(errorLines.length == nerrors, s"Not enough errors recorded.")

val allFiles = (allArgs filter {
arg => !arg.startsWith("-") && (arg.endsWith(".scala") || arg.endsWith(".java"))
}).toList
val expectedErrorsPerFile = allFiles.map(getErrors(_))

// Some compiler errors have an associated source position. Each error
// needs to correspond to a "// error" marker on that line in the source
// file and vice versa.
Expand Down Expand Up @@ -233,7 +249,9 @@ abstract class CompilerTest {
/** Captures the number of nopos-errors in the given file and the number of
* errors with a position, represented as a tuple of source line and number
* of errors on that line. */
case class ErrorsInFile(fileName: String, noposErrorNr: Int, posErrorLinesToNr: List[(Int, Int)])
case class ErrorsInFile(fileName: String, noposErrorNr: Int, posErrorLinesToNr: List[(Int, Int)]) {
def totalErrors = noposErrorNr + posErrorLinesToNr.map(_._2).sum
}

/** Extracts the errors expected for the given neg test file. */
def getErrors(fileName: String): ErrorsInFile = {
Expand Down Expand Up @@ -319,9 +337,9 @@ abstract class CompilerTest {
private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java")

/** Determines what kind of test to run. */
private def testKind(prefixDir: String, xerrors: Int, runTest: Boolean) = {
private def testKind(prefixDir: String, runTest: Boolean) = {
if (runTest) "run"
else if (xerrors > 0) "neg"
else if (isNegTest(prefixDir)) "neg"
else if (prefixDir.endsWith("run" + JFile.separator)) {
log("WARNING: test is being run as pos test despite being in a run directory. " +
"Use runFile/runDir instead of compileFile/compileDir to do a run test")
Expand Down
Loading