Initialize JacoDB and stat executor before starting contest, stop caching machine (#2708)

* Count "ensure JacoDB is initialized and executor is started" in force termination timeout

* Initialize JacoDB and stat executor before starting contest, stop caching machine

Author: IlyaMuravjov

utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt

@@ -47,6 +47,7 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.newSingleThreadContext
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeoutOrNull
+import org.utbot.contest.usvm.createJcContainer
 import org.utbot.contest.usvm.jc.JcContainer
 import org.utbot.contest.usvm.runUsvmGeneration
 import org.utbot.framework.SummariesGenerationType
@@ -128,6 +129,15 @@ fun main(args: Array<String>) {
         // TODO usvm-sbft-merge: Soot is not not used in usvm
         // TestCaseGenerator(listOf(classfileDir), classpathString, dependencyPath, JdkInfoService.provide())
 
+        // Initialize the JacoDB and start executor before contest is started.
+        // This saves the time budget for real work instead of initialization.
+        runBlocking {
+            createJcContainer(
+                tmpDir = tmpDir,
+                classpathFiles = classpathString.split(File.pathSeparator).map { File(it) }
+            ).runner.ensureRunnerAlive()
+        }
+
 //        TODO usvm-sbft-merge: utbot instrumentation not used in usvm
 //        logger.info().measureTime({ "warmup: kotlin reflection :: init" }) {
 //            prepareClass(ConcreteExecutorPool::class.java, "")

utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/ContestUsvm.kt

@@ -88,27 +88,7 @@ fun runUsvmGeneration(
 
     val classpathFiles = classpathString.split(File.pathSeparator).map { File(it) }
 
-    val jcContainer by lazy {
-        JcContainer(
-            usePersistence = false,
-            persistenceDir = tmpDir,
-            classpath = classpathFiles,
-            javaHome = JdkInfoService.provide().path.toFile(),
-            machineOptions = UMachineOptions(
-                // TODO usvm-sbft: if we have less than CONTEST_TEST_EXECUTION_TIMEOUT time left, we should try execute
-                //  with smaller timeout, but instrumentation currently doesn't allow to change timeout for individual runs
-                timeout = generationTimeoutMillisWithoutCodegen.milliseconds - CONTEST_TEST_EXECUTION_TIMEOUT,
-                pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM),
-                pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.COMPLETELY_FAIR,
-                solverType = SolverType.Z3, // TODO: usvm-ksmt: Yices doesn't work on old linux
-            )
-        ) {
-            // TODO usvm-sbft: we may want to tune these JcSettings for contest
-            // TODO: require usePersistence=false for ClassScorer
-            installFeatures(InMemoryHierarchy, Approximations, ClassScorer(TypeScorer, ::scoreClassNode))
-            loadByteCode(classpathFiles)
-        }
-    }
+    val jcContainer by lazy { createJcContainer(tmpDir, classpathFiles) }
 
     val executor by lazy { JcTestExecutor(jcContainer.cp, jcContainer.runner) }
 
@@ -127,6 +107,7 @@ fun runUsvmGeneration(
     }
 
     logger.info { "STARTED COUNTING BUDGET FOR ${cut.classId.name}" }
+    val budgetStartTimeMillis = System.currentTimeMillis()
 
     if (cut.classLoader.javaClass != URLClassLoader::class.java) {
         logger.error("Seems like classloader for cut not valid (maybe it was backported to system): ${cut.classLoader}")
@@ -147,10 +128,10 @@ fun runUsvmGeneration(
         )
     )
 
-    logger.info().measureTime({ "Contest preparation: ensure JacoDB is initialized (NOT counted in time budget)" }) {
+    logger.info().measureTime({ "Contest preparation: ensure JacoDB is initialized (counted in time budget)" }) {
         jcContainer // force init lazy property
     }
-    logger.info().measureTime({ "Contest preparation: ensure executor is started (NOT counted in time budget)" }) {
+    logger.info().measureTime({ "Contest preparation: ensure executor is started (counted in time budget)" }) {
         jcContainer.runner.ensureRunnerAlive()
     }
 
@@ -178,60 +159,73 @@ fun runUsvmGeneration(
 
         val timeStats = mutableMapOf<String, Long>()
 
-        jcContainer.machine.analyzeAsync(
-            forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2,
-            methods = jcMethods,
-            targets = emptyList()
-        ) { state ->
-            val jcExecution = accumulateMeasureTime("executor.execute(${cut.classId.name})", timeStats, state.entrypoint) {
-                runCatching {
-                    executor.execute(
-                        method = state.entrypoint.typedMethod,
-                        state = state,
-                        stringConstants = jcContainer.machine.stringConstants,
-                        classConstants = jcContainer.machine.classConstants
-                    ) ?: return@analyzeAsync
-                }.getOrElse { e ->
-                    logger.error(e) { "executor.execute(${state.entrypoint}) failed" }
-                    return@analyzeAsync
+        val alreadySpentBudgetMillis = System.currentTimeMillis() - budgetStartTimeMillis
+        JcMachine(
+            cp = jcContainer.cp,
+            options = UMachineOptions(
+                // TODO usvm-sbft: if we have less than CONTEST_TEST_EXECUTION_TIMEOUT time left, we should try execute
+                //  with smaller timeout, but instrumentation currently doesn't allow to change timeout for individual runs
+                timeout = generationTimeoutMillisWithoutCodegen.milliseconds - alreadySpentBudgetMillis.milliseconds - CONTEST_TEST_EXECUTION_TIMEOUT,
+                pathSelectionStrategies = listOf(PathSelectionStrategy.CLOSEST_TO_UNCOVERED_RANDOM),
+                pathSelectorFairnessStrategy = PathSelectorFairnessStrategy.COMPLETELY_FAIR,
+                solverType = SolverType.Z3, // TODO: usvm-ksmt: Yices doesn't work on old linux
+            )
+        ).use { jcMachine ->
+            jcMachine.analyzeAsync(
+                forceTerminationTimeout = (generationTimeoutMillisWithoutCodegen + timeBudgetMs) / 2 - alreadySpentBudgetMillis,
+                methods = jcMethods,
+                targets = emptyList()
+            ) { state ->
+                val jcExecution = accumulateMeasureTime("executor.execute(${cut.classId.name})", timeStats, state.entrypoint) {
+                    runCatching {
+                        executor.execute(
+                            method = state.entrypoint.typedMethod,
+                            state = state,
+                            stringConstants = jcMachine.stringConstants,
+                            classConstants = jcMachine.classConstants
+                        ) ?: return@analyzeAsync
+                    }.getOrElse { e ->
+                        logger.error(e) { "executor.execute(${state.entrypoint}) failed" }
+                        return@analyzeAsync
+                    }
+                }
+                val methodId = jcExecution.method.method.toExecutableId(jcContainer.cp)
+                val utExecution = accumulateMeasureTime("JcToUtExecutionConverter.convert(${cut.classId.name})", timeStats, jcExecution.method.method) {
+                    runCatching {
+                        JcToUtExecutionConverter(
+                            jcExecution = jcExecution,
+                            jcClasspath = jcContainer.cp,
+                            idGenerator = utModelIdGenerator,
+                            instructionIdProvider = instructionIdGenerator,
+                            utilMethodProvider = codeGenerator.context.utilMethodProvider
+                        ).convert()
+                            // for some JcExecutions like RD faults we don't construct UtExecutions, converter logs such cases
+                            ?: return@analyzeAsync
+                    }.getOrElse { e ->
+                        logger.error(e) { "JcToUtExecutionConverter.convert(${jcExecution.method.method}) failed" }
+                        return@analyzeAsync
+                    }
                 }
-            }
-            val methodId = jcExecution.method.method.toExecutableId(jcContainer.cp)
-            val utExecution = accumulateMeasureTime("JcToUtExecutionConverter.convert(${cut.classId.name})", timeStats, jcExecution.method.method) {
                 runCatching {
-                    JcToUtExecutionConverter(
-                        jcExecution = jcExecution,
-                        jcClasspath = jcContainer.cp,
-                        idGenerator = utModelIdGenerator,
-                        instructionIdProvider = instructionIdGenerator,
-                        utilMethodProvider = codeGenerator.context.utilMethodProvider
-                    ).convert()
-                        // for some JcExecutions like RD faults we don't construct UtExecutions, converter logs such cases
-                        ?: return@analyzeAsync
+                    val className = Type.getInternalName(methodId.classId.jClass)
+                    val statsForMethod = methodToStats.getValue(methodId)
+                    statsForMethod.testsGeneratedCount++
+                    utExecution.result.exceptionOrNull()?.let { exception ->
+                        statsForMethod.detectedExceptionFqns += exception::class.java.name
+                    }
+                    utExecution.coverage?.let {
+                        statsForClass.updateCoverage(
+                            newCoverage = it,
+                            isNewClass = !statsForClass.testedClassNames.contains(className),
+                            fromFuzzing = utExecution is UtFuzzedExecution
+                        )
+                    }
+                    statsForClass.testedClassNames.add(className)
+                    testsByMethod.getOrPut(methodId) { mutableListOf() } += utExecution
                 }.getOrElse { e ->
-                    logger.error(e) { "JcToUtExecutionConverter.convert(${jcExecution.method.method}) failed" }
-                    return@analyzeAsync
+                    logger.error(e) { "Test generation failed during stats update for $methodId" }
                 }
             }
-            runCatching {
-                val className = Type.getInternalName(methodId.classId.jClass)
-                val statsForMethod = methodToStats.getValue(methodId)
-                statsForMethod.testsGeneratedCount++
-                utExecution.result.exceptionOrNull()?.let { exception ->
-                    statsForMethod.detectedExceptionFqns += exception::class.java.name
-                }
-                utExecution.coverage?.let {
-                    statsForClass.updateCoverage(
-                        newCoverage = it,
-                        isNewClass = !statsForClass.testedClassNames.contains(className),
-                        fromFuzzing = utExecution is UtFuzzedExecution
-                    )
-                }
-                statsForClass.testedClassNames.add(className)
-                testsByMethod.getOrPut(methodId) { mutableListOf() } += utExecution
-            }.getOrElse { e ->
-                logger.error(e) { "Test generation failed during stats update for $methodId" }
-            }
         }
 
         timeStats.forEach { (regionName, timeMillis) ->
@@ -254,6 +248,21 @@ fun runUsvmGeneration(
     statsForClass
 }
 
+fun createJcContainer(
+    tmpDir: File,
+    classpathFiles: List<File>
+) = JcContainer(
+    usePersistence = false,
+    persistenceDir = tmpDir,
+    classpath = classpathFiles,
+    javaHome = JdkInfoService.provide().path.toFile(),
+) {
+    // TODO usvm-sbft: we may want to tune these JcSettings for contest
+    // TODO: require usePersistence=false for ClassScorer
+    installFeatures(InMemoryHierarchy, Approximations, ClassScorer(TypeScorer, ::scoreClassNode))
+    loadByteCode(classpathFiles)
+}
+
 fun JcClasspath.findMethodOrNull(method: ExecutableId): JcMethod? =
     findClass(method.classId.name).declaredMethods.firstOrNull {
         it.name == method.name && it.jcdbSignature == method.jcdbSignature

utbot-junit-contest/src/main/kotlin/org/utbot/contest/usvm/jc/JcContainer.kt

@@ -7,10 +7,8 @@ import org.jacodb.api.JcDatabase
 import org.jacodb.impl.JcSettings
 import org.jacodb.impl.features.classpaths.UnknownClasses
 import org.jacodb.impl.jacodb
-import org.usvm.UMachineOptions
 import org.usvm.instrumentation.executor.UTestConcreteExecutor
 import org.usvm.instrumentation.instrumentation.JcRuntimeTraceInstrumenterFactory
-import org.usvm.machine.JcMachine
 import org.usvm.util.classpathWithApproximations
 import java.io.File
 import kotlin.time.Duration.Companion.seconds
@@ -23,12 +21,10 @@ class JcContainer private constructor(
     persistenceDir: File,
     classpath: List<File>,
     javaHome: File,
-    machineOptions: UMachineOptions,
     builder: JcSettings.() -> Unit,
 ) : AutoCloseable {
     val db: JcDatabase
     val cp: JcClasspath
-    val machine: JcMachine
     val runner: UTestConcreteExecutor
 
     init {
@@ -60,7 +56,6 @@ class JcContainer private constructor(
         }
         this.db = db
         this.cp = cp
-        this.machine = JcMachine(cp, machineOptions)
         this.runner = UTestConcreteExecutor(
             JcRuntimeTraceInstrumenterFactory::class,
             cpPath,
@@ -77,30 +72,27 @@ class JcContainer private constructor(
     override fun close() {
         cp.close()
         db.close()
-        machine.close()
         runner.close()
     }
 
     companion object : AutoCloseable {
         val CONTEST_TEST_EXECUTION_TIMEOUT = 1.seconds
 
-        private val cache = HashMap<Pair<List<File>, UMachineOptions>, JcContainer>()
+        private val cache = HashMap<List<File>, JcContainer>()
 
         operator fun invoke(
             usePersistence: Boolean,
             persistenceDir: File,
             classpath: List<File>,
             javaHome: File,
-            machineOptions: UMachineOptions,
             builder: JcSettings.() -> Unit,
         ): JcContainer {
-            val cacheKey = classpath to machineOptions
-            return cache[cacheKey] ?: run {
+            return cache[classpath] ?: run {
                 // TODO usvm-sbft: right now max cache size is 1, do we need to increase it?
                 logger.info { "JcContainer cache miss" }
                 close()
-                JcContainer(usePersistence, persistenceDir, classpath, javaHome, machineOptions, builder)
-                    .also { cache[cacheKey] = it }
+                JcContainer(usePersistence, persistenceDir, classpath, javaHome, builder)
+                    .also { cache[classpath] = it }
             }
         }