Skip to content

Make TestCaseGenerator use UsvmSymbolicEngine for entire class #2716

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 13, 2023
Merged

Make TestCaseGenerator use UsvmSymbolicEngine for entire class #2716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions utbot-framework/src/main/kotlin/org/utbot/engine/UsvmSymbolicEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,15 @@ package org.utbot.engine

import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.emptyFlow
import org.utbot.framework.plugin.api.ExecutableId
import org.utbot.framework.plugin.api.UtResult

class UsvmSymbolicEngine {

fun generateWithUsvm(): Flow<UtResult> = emptyFlow()
object UsvmSymbolicEngine {
// TODO implement
fun runUsvmGeneration(
methods: List<ExecutableId>,
classpath: String,
timeoutMillis: Long
): Flow<Pair<ExecutableId, UtResult>> =
emptyFlow()
}
56 changes: 32 additions & 24 deletions utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import kotlinx.coroutines.flow.catch
import kotlinx.coroutines.flow.emitAll
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.flowOf
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.yield
Expand All @@ -16,6 +17,7 @@ import mu.KotlinLogging
import org.utbot.common.*
import org.utbot.engine.EngineController
import org.utbot.engine.Mocker
import org.utbot.engine.UsvmSymbolicEngine
import org.utbot.engine.UtBotSymbolicEngine
import org.utbot.engine.util.mockListeners.ForceMockListener
import org.utbot.engine.util.mockListeners.ForceStaticMockListener
Expand Down Expand Up @@ -173,9 +175,10 @@ open class TestCaseGenerator(
methods: List<ExecutableId>,
mockStrategy: MockStrategyApi,
chosenClassesToMockAlways: Set<ClassId> = Mocker.javaDefaultClasses.mapTo(mutableSetOf()) { it.id },
methodsGenerationTimeout: Long = utBotGenerationTimeoutInMillis,
utBotTimeout: Long = utBotGenerationTimeoutInMillis,
userTaintConfigurationProvider: TaintConfigurationProvider? = null,
generate: (engine: UtBotSymbolicEngine) -> Flow<UtResult> = defaultTestFlow(methodsGenerationTimeout)
generate: (engine: UtBotSymbolicEngine) -> Flow<UtResult> = defaultTestFlow(utBotTimeout),
usvmTimeoutMillis: Long = 0,
): List<UtMethodTestSet> = ConcreteExecutor.defaultPool.use { _ -> // TODO: think on appropriate way to close instrumented processes
if (isCanceled()) return@use methods.map { UtMethodTestSet(it) }

Expand All @@ -189,7 +192,7 @@ open class TestCaseGenerator(
return@use methods.map { method -> UtMethodTestSet(method, errors = method2errors.getValue(method)) }

val executionStartInMillis = System.currentTimeMillis()
val executionTimeEstimator = ExecutionTimeEstimator(methodsGenerationTimeout, methods.size)
val executionTimeEstimator = ExecutionTimeEstimator(utBotTimeout, methods.size)

val currentUtContext = utContext

Expand All @@ -200,6 +203,29 @@ open class TestCaseGenerator(
val forceMockListener = ForceMockListener.create(this, conflictTriggers)
val forceStaticMockListener = ForceStaticMockListener.create(this, conflictTriggers)

suspend fun consumeUtResultFlow(utResultFlow: Flow<Pair<ExecutableId, UtResult>>) =
utResultFlow.catch {
logger.error(it) { "Error in flow" }
}
.collect { (executableId, utResult) ->
when (utResult) {
is UtExecution -> {
if (utResult is UtSymbolicExecution &&
(conflictTriggers.triggered(Conflict.ForceMockHappened) ||
conflictTriggers.triggered(Conflict.ForceStaticMockHappened))
) {
utResult.containsMocking = true
}
method2executions.getValue(executableId) += utResult
}

is UtError -> {
method2errors.getValue(executableId).merge(utResult.description, 1, Int::plus)
logger.error(utResult.error) { "UtError occurred" }
}
}
}

runIgnoringCancellationException {
runBlockingWithCancellationPredicate(isCanceled) {
for ((method, controller) in method2controller) {
Expand All @@ -223,27 +249,7 @@ open class TestCaseGenerator(
engineActions.map { engine.apply(it) }
engineActions.clear()

generate(engine)
.catch {
logger.error(it) { "Error in flow" }
}
.collect {
when (it) {
is UtExecution -> {
if (it is UtSymbolicExecution &&
(conflictTriggers.triggered(Conflict.ForceMockHappened) ||
conflictTriggers.triggered(Conflict.ForceStaticMockHappened))
) {
it.containsMocking = true
}
method2executions.getValue(method) += it
}
is UtError -> {
method2errors.getValue(method).merge(it.description, 1, Int::plus)
logger.error(it.error) { "UtError occurred" }
}
}
}
consumeUtResultFlow(generate(engine).map { utResult -> method to utResult })
} catch (e: Exception) {
logger.error(e) {"Error in engine"}
throw e
Expand Down Expand Up @@ -284,6 +290,8 @@ open class TestCaseGenerator(
}
logger.debug("test generator global scope lifecycle check ended")
}

consumeUtResultFlow(UsvmSymbolicEngine.runUsvmGeneration(methods, classpathForEngine, usvmTimeoutMillis))
}
}

Expand Down
17 changes: 4 additions & 13 deletions utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestFlow.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@ import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.emptyFlow
import kotlinx.coroutines.flow.flattenConcat
import kotlinx.coroutines.flow.flowOf
import org.utbot.engine.UsvmSymbolicEngine
import org.utbot.engine.UtBotSymbolicEngine
import org.utbot.framework.UtSettings
import org.utbot.framework.codegen.domain.SymbolicEngineSource

/**
* Constructs [TestFlow] for customization and creates flow producer.
Expand All @@ -18,8 +16,7 @@ fun testFlow(block: TestFlow.() -> Unit): UtBotSymbolicEngine.() -> Flow<UtResul
* Creates default flow that uses [UtSettings] for customization.
*/
fun defaultTestFlow(timeout: Long) = testFlow {
isSymbolicEngineEnabled = true
symbolicEngineType = SymbolicEngineSource.UnitTestBot
isUtBotSymbolicEngineEnabled = true
generationTimeout = timeout
isFuzzingEnabled = UtSettings.useFuzzing
if (generationTimeout > 0) {
Expand All @@ -42,8 +39,7 @@ class TestFlow internal constructor(block: TestFlow.() -> Unit) {
set(value) {
field = maxOf(0, value)
}
var isSymbolicEngineEnabled = true
var symbolicEngineType = SymbolicEngineSource.UnitTestBot
var isUtBotSymbolicEngineEnabled = true
var isFuzzingEnabled = false
var fuzzingValue: Double = 0.1
set(value) {
Expand All @@ -64,7 +60,7 @@ class TestFlow internal constructor(block: TestFlow.() -> Unit) {
return when {
generationTimeout == 0L -> emptyFlow()
isFuzzingEnabled -> {
when (val value = if (isSymbolicEngineEnabled) (fuzzingValue * generationTimeout).toLong() else generationTimeout) {
when (val value = if (isUtBotSymbolicEngineEnabled) (fuzzingValue * generationTimeout).toLong() else generationTimeout) {
0L -> engine.traverse()
generationTimeout -> engine.fuzzing(System.currentTimeMillis() + value)
else -> flowOf(
Expand All @@ -73,12 +69,7 @@ class TestFlow internal constructor(block: TestFlow.() -> Unit) {
).flattenConcat()
}
}
isSymbolicEngineEnabled -> {
when (symbolicEngineType) {
SymbolicEngineSource.UnitTestBot -> engine.traverse()
SymbolicEngineSource.Usvm -> UsvmSymbolicEngine().generateWithUsvm()
}
}
isUtBotSymbolicEngineEnabled -> engine.traverse()
else -> emptyFlow()
}
}
Expand Down
17 changes: 13 additions & 4 deletions utbot-framework/src/main/kotlin/org/utbot/framework/process/EngineProcessMain.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import org.utbot.framework.codegen.domain.ParametrizedTestSource
import org.utbot.framework.codegen.domain.ProjectType
import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
import org.utbot.framework.codegen.domain.SymbolicEngineSource
import org.utbot.framework.codegen.domain.SymbolicEngineSource.*
import org.utbot.framework.codegen.domain.testFrameworkByName
import org.utbot.framework.codegen.generator.AbstractCodeGenerator
import org.utbot.framework.codegen.generator.CodeGeneratorParams
Expand Down Expand Up @@ -114,10 +115,17 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
val methods: List<ExecutableId> = kryoHelper.readObject(params.methods)
logger.debug()
.measureTime({ "starting generation for ${methods.size} methods, starting with ${methods.first()}" }) {
val symbolicEngineSource = SymbolicEngineSource.valueOf(params.symbolicEngineType)
val utBotTimeout = when {
!params.isSymbolicEngineEnabled -> params.timeout
symbolicEngineSource == UnitTestBot -> params.timeout
params.isFuzzingEnabled -> (params.fuzzingValue * params.timeout).toLong()
else -> 0L
}

val generateFlow = testFlow {
generationTimeout = params.generationTimeout
isSymbolicEngineEnabled = params.isSymbolicEngineEnabled
symbolicEngineType = SymbolicEngineSource.valueOf(params.symbolicEngineType)
generationTimeout = utBotTimeout
isUtBotSymbolicEngineEnabled = params.isSymbolicEngineEnabled && symbolicEngineSource == UnitTestBot
isFuzzingEnabled = params.isFuzzingEnabled
fuzzingValue = params.fuzzingValue
}
Expand All @@ -130,9 +138,10 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
methods,
MockStrategyApi.valueOf(params.mockStrategy),
kryoHelper.readObject(params.chosenClassesToMockAlways),
params.timeout,
utBotTimeout,
userTaintConfigurationProvider,
generate = generateFlow,
usvmTimeoutMillis = params.generationTimeout - utBotTimeout,
)
.summarizeAll(Paths.get(params.searchDirectory), null)
.filterNot { it.executions.isEmpty() && it.errors.isEmpty() }
Expand Down