Refactor bench (fix #26)

This commit splits project `bench` into multiple projects:
`benchJavac18`, `benchRsc`, `benchScalac211` and `benchScalac212`.
This provides additional flexibility and fine-grained control over
target compiler versions.

The main motivation for this change is the fact that neither Scala 2.11
nor Scala 2.12 clearly dominate one another on re2s. Scala 2.11 is
faster in cold mode, whereas Scala 2.12 is faster in hot mode.

While refactoring the benchmarking infrastructure to accommodate this
change, I've also retired CliBench. It is confusing to have multiple
benchmarks that test the same thing (ColdXxxYyy and CliXxxYyy),
so I picked the one that's simpler, i.e. ColdXxxYyy powered by JMH.
RscNativeTypecheck is still powered by CliBench-like logic, but it has
been merged directly into that particular benchmark.

I am not completely happy with the astonishing degree of duplication
between benchScalac211 and benchScalac212, but my sbt skills don't
allow me to remove this duplication in a satisfying manner. (I'm aware
of ++, but I want Scala versions to be part of the project name, not
an optional prefix to an sbt invocation.)

bench/javac18/src/main/scala/rsc/bench/FileFixtures.scala

@@ -0,0 +1,22 @@
+// Copyright (c) 2017 Twitter, Inc.
+// Licensed under the Apache License, Version 2.0 (see LICENSE.md).
+package rsc.bench
+
+import java.io._
+import java.nio.file.Files
+import scala.collection.JavaConverters._
+
+trait FileFixtures {
+  lazy val buildRoot: File = {
+    BuildInfo.sourceRoot
+  }
+
+  lazy val re2jDir: File = {
+    new File(s"$buildRoot/examples/re2j/src/main/java/java/util/regex")
+  }
+
+  lazy val re2jFiles: List[File] = {
+    val stream = Files.newDirectoryStream(re2jDir.toPath)
+    stream.asScala.map(_.toFile).toList
+  }
+}

bench/jvm/src/main/scala/rsc/bench/JavacCompile.scala → bench/javac18/src/main/scala/rsc/bench/JavacCompile.scala

@@ -8,27 +8,17 @@ import javax.tools._
 import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.annotations.Mode._
 import rsc.bench.JavacCompile._
-import rsc.tests._
 
 object JavacCompile {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures {
+  class BenchmarkState extends FileFixtures {
     val compiler = ToolProvider.getSystemJavaCompiler()
     val re2jFilenames = re2jFiles.map(_.getAbsolutePath)
     val outDir = Files.createTempDirectory("javac_")
     val options = List("-d", outDir.toString) ++ re2jFilenames
   }
 }
 
-object CliJavacCompile {
-  def main(args: Array[String]): Unit = {
-    val Array(expectedJavacVersion) = args
-    val bs = new JavacCompile.BenchmarkState
-    val command = List("javac") ++ bs.options
-    CliBench.run(command, runs = 100)
-  }
-}
-
 trait JavacCompile {
   def runImpl(bs: BenchmarkState): Unit = {
     val exitCode = bs.compiler.run(null, null, null, bs.options: _*)

bench/rsc/jvm/src/main/scala/rsc/bench/RscNativeTypecheck.scala

@@ -0,0 +1,42 @@
+// Copyright (c) 2017 Twitter, Inc.
+// Licensed under the Apache License, Version 2.0 (see LICENSE.md).
+package rsc.bench
+
+import rsc.bench.RscNativeTypecheck._
+
+object RscNativeTypecheck {
+  class BenchmarkState extends FileFixtures
+}
+
+object CliRscNativeTypecheck {
+  def main(args: Array[String]): Unit = {
+    val Array(out) = args
+    val bs = new BenchmarkState
+    val fs = bs.re2sRscFiles.map(_.toString)
+    val options = List("-Ystop-after:typecheck") ++ fs
+    run(List(out) ++ options, runs = 100, iters = 1)
+    run(List(out) ++ options, runs = 1, iters = 100)
+  }
+
+  private def run(command: List[String], runs: Int, iters: Int): Unit = {
+    println(s"Running ${command.mkString(" ")} $runs x $iters times...")
+    val times = 1.to(runs).map { i =>
+      val start = System.nanoTime()
+      val process = new java.lang.ProcessBuilder()
+      process.command((command ++ List("--iters", iters.toString)): _*)
+      process.directory(rsc.bench.BuildInfo.sourceRoot)
+      process.redirectOutput(ProcessBuilder.Redirect.INHERIT)
+      process.redirectError(ProcessBuilder.Redirect.INHERIT)
+      val exitcode = process.start().waitFor()
+      if (exitcode != 0) {
+        sys.error(s"Command has failed with code $exitcode")
+      }
+      val end = System.nanoTime()
+      val result = 1.0 * (end - start) / 1000000
+      println(s"Run $i: $result ms")
+      result
+    }
+    val result = times.sum / runs
+    println(s"Average: " + result + " ms")
+  }
+}

bench/jvm/src/main/scala/rsc/bench/RscOutline.scala → bench/rsc/jvm/src/main/scala/rsc/bench/RscOutline.scala

@@ -7,16 +7,15 @@ import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.annotations.Mode._
 import rsc.bench.RscOutline._
 import rsc.semantics._
-import rsc.tests._
 
 object RscOutline {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures
+  class BenchmarkState extends RscFixtures with FileFixtures
 }
 
 trait RscOutline {
   def runImpl(bs: BenchmarkState): Unit = {
-    val compiler = bs.mkCompiler("-Ystop-after:outline", bs.re2sFiles)
+    val compiler = bs.mkCompiler("-Ystop-after:outline", bs.re2sRscFiles)
     compiler.run()
     val problems = compiler.reporter.problems
     if (problems.nonEmpty) {

bench/jvm/src/main/scala/rsc/bench/RscParse.scala → bench/rsc/jvm/src/main/scala/rsc/bench/RscParse.scala

@@ -10,12 +10,11 @@ import rsc.lexis._
 import rsc.parse._
 import rsc.report._
 import rsc.settings._
-import rsc.tests._
 
 object RscParse {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures {
-    val settings = Settings.parse(re2sFiles.map(_.toString)).get
+  class BenchmarkState extends FileFixtures {
+    val settings = Settings.parse(re2sRscFiles.map(_.toString)).get
     val reporter = StoreReporter(settings)
     val inputs = settings.ins.map(Input.apply).toArray
   }

bench/jvm/src/main/scala/rsc/bench/RscScan.scala → bench/rsc/jvm/src/main/scala/rsc/bench/RscScan.scala

@@ -10,12 +10,11 @@ import rsc.lexis._
 import rsc.report._
 import rsc.scan._
 import rsc.settings._
-import rsc.tests._
 
 object RscScan {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures {
-    val settings = Settings.parse(re2sFiles.map(_.toString)).get
+  class BenchmarkState extends FileFixtures {
+    val settings = Settings.parse(re2sRscFiles.map(_.toString)).get
     val reporter = StoreReporter(settings)
     val inputs = settings.ins.map(Input.apply).toArray
   }

bench/jvm/src/main/scala/rsc/bench/RscSchedule.scala → bench/rsc/jvm/src/main/scala/rsc/bench/RscSchedule.scala

@@ -7,16 +7,15 @@ import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.annotations.Mode._
 import rsc.bench.RscSchedule._
 import rsc.semantics._
-import rsc.tests._
 
 object RscSchedule {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures
+  class BenchmarkState extends RscFixtures with FileFixtures
 }
 
 trait RscSchedule {
   def runImpl(bs: BenchmarkState): Unit = {
-    val compiler = bs.mkCompiler("-Ystop-after:schedule", bs.re2sFiles)
+    val compiler = bs.mkCompiler("-Ystop-after:schedule", bs.re2sRscFiles)
     compiler.run()
     val problems = compiler.reporter.problems
     if (problems.nonEmpty) {

bench/jvm/src/main/scala/rsc/bench/RscScope.scala → bench/rsc/jvm/src/main/scala/rsc/bench/RscScope.scala

@@ -7,16 +7,15 @@ import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.annotations.Mode._
 import rsc.bench.RscScope._
 import rsc.semantics._
-import rsc.tests._
 
 object RscScope {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures
+  class BenchmarkState extends RscFixtures with FileFixtures
 }
 
 trait RscScope {
   def runImpl(bs: BenchmarkState): Unit = {
-    val compiler = bs.mkCompiler("-Ystop-after:scope", bs.re2sFiles)
+    val compiler = bs.mkCompiler("-Ystop-after:scope", bs.re2sRscFiles)
     compiler.run()
     val problems = compiler.reporter.problems
     if (problems.nonEmpty) {

bench/jvm/src/main/scala/rsc/bench/RscTypecheck.scala → bench/rsc/jvm/src/main/scala/rsc/bench/RscTypecheck.scala

@@ -7,40 +7,15 @@ import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.annotations.Mode._
 import rsc.bench.RscTypecheck._
 import rsc.semantics._
-import rsc.tests._
 
 object RscTypecheck {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures
-}
-
-object CliRscTypecheck {
-  def main(args: Array[String]): Unit = {
-    val Array(classpath) = args
-    val bs = new ScalacCompile.BenchmarkState
-    val fs = bs.re2sFiles.map(_.toString)
-    val options = List("-Ystop-after:typecheck") ++ fs
-    val command = List("scala", "-cp", classpath, "rsc.bench.Main") ++ options
-    CliBench.run(command, runs = 100)
-  }
-}
-
-object CliRscNativeTypecheck {
-  def main(args: Array[String]): Unit = {
-    val Array(out) = args
-    val bs = new ScalacCompile.BenchmarkState
-    val fs = bs.re2sFiles.map(_.toString)
-    val options = List("-Ystop-after:typecheck") ++ fs
-    val command1 = List(out) ++ options ++ List("--runs", "1")
-    CliBench.run(command1, runs = 100)
-    val command2 = List(out) ++ options ++ List("--runs", "100")
-    CliBench.run(command2, runs = 1)
-  }
+  class BenchmarkState extends RscFixtures with FileFixtures
 }
 
 trait RscTypecheck {
   def runImpl(bs: BenchmarkState): Unit = {
-    val compiler = bs.mkCompiler("-Ystop-after:typecheck", bs.re2sFiles)
+    val compiler = bs.mkCompiler("-Ystop-after:typecheck", bs.re2sRscFiles)
     compiler.run()
     val problems = compiler.reporter.problems
     if (problems.nonEmpty) {

bench/shared/src/main/scala/rsc/bench/Main.scala → bench/rsc/native/src/main/scala/rsc/bench/Main.scala

@@ -13,25 +13,25 @@ object Main {
   }
 
   def run(args: Array[String]): Int = {
-    case class BenchSettings(args1: List[String] = Nil, runs: Int = 1)
+    case class BenchSettings(args1: List[String] = Nil, iters: Int = 1)
     def loop(
         settings: BenchSettings,
         allowOptions: Boolean,
         args: List[String]): BenchSettings = {
       args match {
         case "--" +: rest =>
           loop(settings, false, args)
-        case "--runs" +: s_runs +: rest if allowOptions =>
-          loop(settings.copy(runs = s_runs.toInt), true, rest)
+        case "--iters" +: s_iters +: rest if allowOptions =>
+          loop(settings.copy(iters = s_iters.toInt), true, rest)
         case arg +: rest =>
           loop(settings.copy(args1 = settings.args1 :+ arg), allowOptions, rest)
         case Nil =>
           settings
       }
     }
-    val BenchSettings(args1, runs) = loop(BenchSettings(), true, args.toList)
+    val BenchSettings(args1, iters) = loop(BenchSettings(), true, args.toList)
 
-    for (i <- 1 to runs) {
+    for (i <- 1 to iters) {
       Settings.parse(args1) match {
         case Some(settings) =>
           val reporter = ConsoleReporter(settings)

bench/rsc/shared/src/main/scala/rsc/bench/FileFixtures.scala

@@ -0,0 +1,26 @@
+// Copyright (c) 2017 Twitter, Inc.
+// Licensed under the Apache License, Version 2.0 (see LICENSE.md).
+package rsc.bench
+
+import java.io._
+import java.nio.file.Files
+import scala.collection.JavaConverters._
+
+trait FileFixtures {
+  lazy val buildRoot: File = {
+    BuildInfo.sourceRoot
+  }
+
+  lazy val re2sDir: File = {
+    new File(s"$buildRoot/examples/re2s/src/main/scala/java/util/regex")
+  }
+
+  lazy val re2sRscFiles: List[File] = {
+    val stream = Files.newDirectoryStream(re2sDir.toPath)
+    stream.asScala.map(_.toFile).toList :+ stdlibFile
+  }
+
+  lazy val stdlibFile: File = {
+    new File(s"$buildRoot/stdlib/src/main/scala/Stdlib.scala")
+  }
+}

bench/rsc/shared/src/main/scala/rsc/bench/RscFixtures.scala

@@ -0,0 +1,19 @@
+// Copyright (c) 2017 Twitter, Inc.
+// Licensed under the Apache License, Version 2.0 (see LICENSE.md).
+package rsc.bench
+
+import rsc.Compiler
+import rsc.report._
+import rsc.settings._
+
+trait RscFixtures {
+  def mkCompiler(args: Any*): Compiler = {
+    val options = args.flatMap {
+      case seq: Seq[_] => seq.map(_.toString)
+      case other => List(other.toString)
+    }
+    val settings = Settings.parse(options.toList).get
+    val reporter = StoreReporter(settings)
+    Compiler(settings, reporter)
+  }
+}

bench/scalac211/src/main/scala/rsc/bench/FileFixtures.scala

@@ -0,0 +1,22 @@
+// Copyright (c) 2017 Twitter, Inc.
+// Licensed under the Apache License, Version 2.0 (see LICENSE.md).
+package rsc.bench
+
+import java.io._
+import java.nio.file.Files
+import scala.collection.JavaConverters._
+
+trait FileFixtures {
+  lazy val buildRoot: File = {
+    BuildInfo.sourceRoot
+  }
+
+  lazy val re2sDir: File = {
+    new File(s"$buildRoot/examples/re2s/src/main/scala/java/util/regex")
+  }
+
+  lazy val re2sScalacFiles: List[File] = {
+    val stream = Files.newDirectoryStream(re2sDir.toPath)
+    stream.asScala.map(_.toFile).toList
+  }
+}

bench/jvm/src/main/scala/rsc/bench/ScalacCompile.scala → bench/scalac211/src/main/scala/rsc/bench/ScalacCompile.scala

@@ -9,22 +9,10 @@ import org.openjdk.jmh.annotations.Mode._
 import scala.tools.nsc._
 import scala.tools.nsc.reporters._
 import rsc.bench.ScalacCompile._
-import rsc.tests._
 
 object ScalacCompile {
   @State(Scope.Benchmark)
-  class BenchmarkState extends RscFixtures
-}
-
-object CliScalacCompile {
-  def main(args: Array[String]): Unit = {
-    val Array(expectedScalacVersion) = args
-    val bs = new ScalacCompile.BenchmarkState
-    val outdir = Files.createTempDirectory("scalac_").toString
-    val fs = bs.re2sFiles.init.map(_.toString)
-    val command = List("scalac", "-d", outdir, "-usejavacp") ++ fs
-    CliBench.run(command, runs = 100)
-  }
+  class BenchmarkState extends FileFixtures
 }
 
 trait ScalacCompile {
@@ -35,7 +23,7 @@ trait ScalacCompile {
     val reporter = new StoreReporter
     val global = Global(settings, reporter)
     val run = new global.Run
-    run.compile(bs.re2sFiles.init.map(_.toString))
+    run.compile(bs.re2sScalacFiles.map(_.toString))
     if (reporter.hasErrors) {
       reporter.infos.foreach(println)
       sys.error("compile failed")