Add test without jmh, add junit dataset

benchmarks/build.gradle.kts

@@ -1,8 +1,6 @@
 plugins {
     java
     kotlin("jvm") version "1.9.20"
-    kotlin("plugin.allopen") version "1.9.20"
-    id("me.champeau.jmh") version "0.7.2"
 }
 
 repositories {
@@ -11,61 +9,17 @@ repositories {
 }
 
 dependencies {
-    //benchmarks tool
-    testImplementation("org.jetbrains.kotlin:kotlin-test")
-    implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.10")
+    testImplementation("org.junit.jupiter:junit-jupiter-api:5.11.0")
     //compared projects
     // 1. for ucfs
     implementation(project(":solver"))
     implementation(project(":generator"))
     implementation(project(":examples"))
     // 2. for antlr
     implementation("org.antlr:antlr4:4.13.1")
+    implementation(kotlin("stdlib-jdk8"))
 }
 
-fun getArgs(strFolder: String): List<String> {
-    val resourcesDir = File(strFolder)
-    val files = resourcesDir.listFiles()!!
-    return files.map { it.toString() }.sorted().toList()
-}
-jmh {
-    duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE
-    warmupIterations = 5
-    warmup = "1s"
-    iterations = 10
-    timeOnIteration = "2s"
-    verbosity = "EXTRA"
-    benchmarkMode.addAll("AverageTime")
-    failOnError = false
-    resultFormat = "CSV"
-    jvmArgs.addAll("-Xmx128g")
-    val buildDir = project.layout.buildDirectory.get().toString()
-    humanOutputFile = project.file("${buildDir}/reports/jmh/human.txt") // human-readable output file
-    resultsFile = project.file("${buildDir}/reports/jmh/results.txt") // results file
-    profilers = listOf("gc")
-
-
-    //configure files
-    val dataset = "dataset"
-    if (!hasProperty(dataset)) {
-        println("BENCHMARKS FAILED! Set dataset folder by property '$dataset'")
-    } else {
-        val params = objects.listProperty<String>().value(getArgs(property(dataset).toString()))
-        benchmarkParameters.put("fileName", params)
-    }
-
-    //filter on tools
-    val tools = "toolName"
-    if (hasProperty(tools)) {
-        println("Run benchmarks for: .*${property(tools)}.*")
-        includes = listOf(".*${property(tools)}.*")
-    }
-
-}
-
-
-allOpen {
-    annotation("org.openjdk.jmh.annotations.State")
-}
-
-kotlin { jvmToolchain(11) }
+kotlin {
+    jvmToolchain(17)
+}

benchmarks/src/jmh/kotlin/org/Benchmarks.kt → benchmarks/src/main/kotlin/Benchmarks.kt

@@ -1,7 +1,5 @@
-package org
-
-import java8.JavaLexer
 import java8.JavaToken
+import org.ucfs.JavaLexer
 import org.ucfs.input.*
 import org.ucfs.rsm.symbol.Term
 import java.io.StringReader

benchmarks/src/jmh/kotlin/org/Antlr.kt → benchmarks/src/test/kotlin/AntlrBenchmark.kt

@@ -1,25 +1,23 @@
-package org
-
-import kotlinx.benchmark.*
 import org.antlr.Java8Lexer
 import org.antlr.Java8Parser
 import org.antlr.v4.runtime.CharStreams
 import org.antlr.v4.runtime.CommonTokenStream
 
+class AntlrBenchmark: TimeParsingBenchmark() {
 
-@State(Scope.Benchmark)
-class Antlr : BaseBench(){
+    override fun getShortName(): String = "Antlr"
 
-    @Benchmark
-    fun measureAntlr(blackhole: Blackhole) {
+    @Override
+    override fun parse(text: String) {
         val antlrParser =
             Java8Parser(
                 CommonTokenStream(
                     Java8Lexer(
-                        CharStreams.fromString(fileContents)
+                        CharStreams.fromString(text)
                     )
                 )
             )
-        blackhole.consume(antlrParser.compilationUnit())
+        antlrParser.compilationUnit()
     }
-}
+
+}

benchmarks/src/test/kotlin/OfflineUcfsBenchmark.kt

@@ -0,0 +1,11 @@
+import org.ucfs.input.LinearInputLabel
+
+class OfflineUcfsBenchmark : TimeParsingBenchmark() {
+    override fun getShortName(): String = "UcfsOff"
+
+    override fun parse(text: String) {
+        val parser = org.ucfs.Java8Parser<Int, LinearInputLabel>()
+        parser.setInput(getTokenStream(text))
+        parser.parse()
+    }
+}

benchmarks/src/test/kotlin/OnlineUcfsBenchmark.kt

@@ -0,0 +1,13 @@
+import java8.Java8
+import org.ucfs.parser.Gll
+
+class OnlineUcfsBenchmark : TimeParsingBenchmark() {
+    override fun getShortName(): String = "UcfsOn"
+
+    override fun parse(text: String) {
+        val startState = Java8().rsm
+        val tokens = getTokenStream(text)
+        val gll = Gll.gll(startState, tokens)
+        gll.parse()
+    }
+}

benchmarks/src/test/kotlin/RecoveryOfflineUcfsBenchmark.kt

@@ -0,0 +1,10 @@
+import org.ucfs.input.LinearInputLabel
+
+class RecoveryOfflineUcfsBenchmark : TimeParsingBenchmark() {
+    override fun getShortName(): String = "RecUcfsOff"
+    override fun parse(text: String) {
+        val parser = org.ucfs.Java8ParserRecovery<Int, LinearInputLabel>()
+        parser.setInput(getTokenStream(text))
+        parser.parse()
+    }
+}

benchmarks/src/test/kotlin/RecoveryOnlineUcfsBenchmark.kt

@@ -0,0 +1,15 @@
+import java8.Java8
+import org.ucfs.parser.Gll
+
+class RecoveryOnlineUcfsBenchmark : TimeParsingBenchmark() {
+
+    override fun getShortName(): String = "RecUcfsOn"
+
+
+    override fun parse(text: String) {
+        val startState = Java8().rsm
+        val tokens = getTokenStream(text)
+        val gll = Gll.gll(startState, tokens)
+        gll.parse()
+    }
+}

benchmarks/src/test/kotlin/TimeParsingBenchmark.kt

@@ -0,0 +1,54 @@
+import org.junit.jupiter.api.DynamicTest
+import org.junit.jupiter.api.DynamicTest.dynamicTest
+import org.junit.jupiter.api.TestFactory
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.name
+
+abstract class TimeParsingBenchmark {
+
+
+    val repeatCount: Int = 5
+    abstract fun getShortName(): String
+
+    private fun runTimeTest(fileName: String, text: String) {
+        var result: Long = 0
+        for(i in 0..repeatCount) {
+            val startTime = System.currentTimeMillis()
+            parse(text)
+            result += System.currentTimeMillis() - startTime
+        }
+        result /= repeatCount
+        val message = "$fileName,$result"
+        println(message)
+        File("${getShortName()}.csv").writeText(message)
+    }
+
+    abstract fun parse(text: String)
+
+    private fun getResourceFolder(): String = Path.of("java", "correct").toString()
+
+
+    private fun getResource(resourceFolder: String): Path {
+        val res = TimeParsingBenchmark::class.java.getResource(resourceFolder)
+            ?: throw RuntimeException("No resource '$resourceFolder'")
+        return Path.of(res.toURI())
+    }
+
+    @TestFactory
+    fun timeTest(): Collection<DynamicTest> {
+        return getTests(getResource(getResourceFolder()), ::runTimeTest)
+    }
+
+    private fun getTests(folder: Path, run: (String, String) -> Unit): Collection<DynamicTest> {
+        return Files.list(folder).map { file ->
+            dynamicTest(file.fileName.toString()) {
+                val source = file.toFile().readText()
+                run(file.name, source)
+            }
+        }.toList()
+    }
+
+}
+