Skip to content

Unified error and exception clusters for all UtExecutions #1250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Nov 8, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 20 additions & 54 deletions utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,6 @@ import java.io.File
import java.nio.file.Path
import java.nio.file.Paths
import mu.KotlinLogging
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
import org.utbot.framework.plugin.api.UtExecutionSuccess
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
import org.utbot.framework.plugin.api.UtImplicitlyThrownException
import org.utbot.framework.plugin.api.UtOverflowFailure
import org.utbot.framework.plugin.api.UtSandboxFailure
import org.utbot.framework.plugin.api.UtStreamConsumingFailure
import org.utbot.framework.plugin.api.UtTimeoutException
import org.utbot.framework.plugin.api.util.humanReadableName
import org.utbot.framework.plugin.api.util.jClass
import org.utbot.fuzzer.FuzzedMethodDescription
import org.utbot.fuzzer.FuzzedValue
Expand Down Expand Up @@ -222,9 +213,8 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
testSet: UtMethodTestSet
): List<UtExecutionCluster> {
val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
val executionsProducedByFuzzer = testSet.executions.filterIsInstance<UtFuzzedExecution>()
val successfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
val unsuccessfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
val testSetWithFuzzedExecutions = prepareTestSetWithFuzzedExecutions(testSet)
val executionsProducedByFuzzer = testSetWithFuzzedExecutions.executions as List<UtFuzzedExecution>

if (executionsProducedByFuzzer.isNotEmpty()) {
executionsProducedByFuzzer.forEach { utExecution ->
Expand All @@ -239,44 +229,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
)
}.firstOrNull()
} catch (t: Throwable) {
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behaviour
null
}

utExecution.testMethodName = testMethodName?.testName
utExecution.displayName = testMethodName?.displayName
utExecution.summary = testMethodName?.javaDoc

when (utExecution.result) {
is UtConcreteExecutionFailure,
is UtExplicitlyThrownException,
is UtImplicitlyThrownException,
is UtOverflowFailure,
is UtSandboxFailure,
is UtTimeoutException,
is UtStreamConsumingFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution)
}
}

if (successfulFuzzerExecutions.isNotEmpty()) {
val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet)

clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(clusterHeader, null),
successfulFuzzerExecutions
)
)
}

if (unsuccessfulFuzzerExecutions.isNotEmpty()) {
val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet)
val clusteredExecutions = groupFuzzedExecutions(testSetWithFuzzedExecutions)

clusteredExecutions.forEach {
clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(clusterHeader, null),
unsuccessfulFuzzerExecutions
UtClusterInfo(it.header),
it.executions
)
)
}
Expand All @@ -285,20 +252,6 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
return clustersToReturn.toList()
}

private fun buildFuzzerClusterHeaderForSuccessfulExecutions(testSet: UtMethodTestSet): String {
val commentPrefix = "FUZZER:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"

return "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix"
}

private fun buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet: UtMethodTestSet): String {
val commentPrefix = "FUZZER:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"

return "$commentPrefix ${ExecutionGroup.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS} $commentPostfix"
}

/** Filter and copies executions with non-empty paths. */
private fun prepareTestSetForByteCodeAnalysis(testSet: UtMethodTestSet): UtMethodTestSet {
val executions =
Expand All @@ -314,6 +267,19 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
)
}

/** Filter and copies fuzzed executions. */
private fun prepareTestSetWithFuzzedExecutions(testSet: UtMethodTestSet): UtMethodTestSet {
val executions = testSet.executions.filterIsInstance<UtFuzzedExecution>()

return UtMethodTestSet(
method = testSet.method,
executions = executions,
jimpleBody = testSet.jimpleBody,
errors = testSet.errors,
clustersInfo = testSet.clustersInfo
)
}

/** Filter and copies executions with non-empty paths. */
private fun prepareTestSetWithEmptyPaths(testSet: UtMethodTestSet): UtMethodTestSet {
val executions =
Expand Down
90 changes: 59 additions & 31 deletions utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,20 @@ package org.utbot.summary

import org.utbot.framework.plugin.api.Step
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
import org.utbot.framework.plugin.api.UtSymbolicExecution
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtExecutionResult
import org.utbot.framework.plugin.api.UtExecutionSuccess
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
import org.utbot.framework.plugin.api.UtImplicitlyThrownException
import org.utbot.framework.plugin.api.UtOverflowFailure
import org.utbot.framework.plugin.api.UtMethodTestSet
import org.utbot.framework.plugin.api.UtOverflowFailure
import org.utbot.framework.plugin.api.UtSandboxFailure
import org.utbot.framework.plugin.api.UtStreamConsumingFailure
import org.utbot.framework.plugin.api.UtSymbolicExecution
import org.utbot.framework.plugin.api.UtTimeoutException
import org.utbot.framework.plugin.api.util.humanReadableName
import org.utbot.framework.plugin.api.util.isCheckedException
import org.utbot.fuzzer.UtFuzzedExecution
import org.utbot.summary.UtSummarySettings.MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING
import org.utbot.summary.clustering.MatrixUniqueness
import org.utbot.summary.clustering.SplitSteps
Expand All @@ -29,7 +31,7 @@ class TagGenerator {

if (clusteredExecutions.isNotEmpty()) {
val listOfSplitSteps = clusteredExecutions.map {
val mUniqueness = MatrixUniqueness(it.executions)
val mUniqueness = MatrixUniqueness(it.executions as List<UtSymbolicExecution>)
mUniqueness.splitSteps()
}

Expand Down Expand Up @@ -64,7 +66,7 @@ class TagGenerator {
traceTagClusters.add(
TraceTagCluster(
cluster.header,
generateExecutionTags(cluster.executions, splitSteps),
generateExecutionTags(cluster.executions as List<UtSymbolicExecution>, splitSteps),
TraceTagWithoutExecution(
commonStepsInCluster.toList(),
cluster.executions.first().result,
Expand All @@ -88,51 +90,67 @@ private fun generateExecutionTags(executions: List<UtSymbolicExecution>, splitSt


/**
* Splits executions into clusters
* By default there is 5 types of clusters:
* Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
* These are split by the type of execution result
* Splits executions with empty paths into clusters.
*
* @return clustered executions
* @return clustered executions.
*/
fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionCluster> {
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
val clusters = mutableListOf<ExecutionCluster>()
val commentPrefix = "OTHER:"
val commentPrefix = "OTHER:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"

val grouped = methodExecutions.groupBy { it.result.clusterKind() }

val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
if (successfulExecutions.isNotEmpty()) {
clusters += SuccessfulExecutionCluster(
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
successfulExecutions.toList())
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
successfulExecutions.toList()
)
}

clusters += grouped
.filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
.map { (suffixId, group) ->
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
}
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
return clusters
}

/**
* Splits fuzzed executions into clusters.
*
* @return clustered executions.
*/
fun groupFuzzedExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
val methodExecutions = testSet.executions.filterIsInstance<UtFuzzedExecution>()
val clusters = mutableListOf<ExecutionCluster>()
val commentPrefix = "FUZZER:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"

val grouped = methodExecutions.groupBy { it.result.clusterKind() }

val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
if (successfulExecutions.isNotEmpty()) {
clusters += SuccessfulExecutionCluster(
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
successfulExecutions.toList()
)
}

clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
return clusters
}

/**
* Splits executions produced by symbolic execution engine into clusters
* By default there is 5 types of clusters:
* Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
* These are split by the type of execution result
* Splits symbolic executions into clusters.
*
* If Success cluster has more than MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING execution
* then clustering algorithm splits those into more clusters
* If Success cluster has more than [MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING] execution
* then clustering algorithm splits those into more clusters.
*
* @return clustered executions
* @return clustered executions.
*/
private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
val clusters = mutableListOf<ExecutionCluster>()
val commentPrefix = "SYMBOLIC EXECUTION:"
val commentPrefix = "SYMBOLIC EXECUTION:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"

val grouped = methodExecutions.groupBy { it.result.clusterKind() }
Expand Down Expand Up @@ -161,11 +179,21 @@ private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster
}
}

clusters += grouped
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
return clusters
}

private fun addClustersOfFailedExecutions(
grouped: Map<ExecutionGroup, List<UtExecution>>,
commentPrefix: String,
commentPostfix: String
): List<FailedExecutionCluster> {
val clusters = grouped
.filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
.map { (suffixId, group) ->
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
}
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
}

return clusters
}

Expand Down Expand Up @@ -197,18 +225,18 @@ private fun UtExecutionResult.clusterKind() = when (this) {
/**
* Structure used to represent execution cluster with header
*/
sealed class ExecutionCluster(var header: String, val executions: List<UtSymbolicExecution>)
sealed class ExecutionCluster(var header: String, val executions: List<UtExecution>)

/**
* Represents successful execution cluster
*/
private class SuccessfulExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
private class SuccessfulExecutionCluster(header: String, executions: List<UtExecution>) :
ExecutionCluster(header, executions)

/**
* Represents failed execution cluster
*/
private class FailedExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
private class FailedExecutionCluster(header: String, executions: List<UtExecution>) :
ExecutionCluster(header, executions)

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,27 @@ package org.utbot.summary

object UtSummarySettings {
/**
* If True test comments will be generated
* If True test comments will be generated.
*/
var GENERATE_COMMENTS = true

/**
* If True cluster comments will be generated
* If True cluster comments will be generated.
*/
var GENERATE_CLUSTER_COMMENTS = true

/**
* If True names for tests will be generated
* If True names for tests will be generated.
*/
var GENERATE_NAMES = true

/**
* If True display names for tests will be generated
* If True display names for tests will be generated.
*/
var GENERATE_DISPLAY_NAMES = true

/**
* generate display name in from -> to style
* If True display name in from -> to style will be generated.
*/
var GENERATE_DISPLAYNAME_FROM_TO_STYLE = true

Expand All @@ -34,19 +34,21 @@ object UtSummarySettings {

/**
* Sets minimum number of successful execution
* for applying the clustering algorithm
* for applying the clustering algorithm.
*/
const val MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING: Int = 4

/**
* DBSCAN hyperparameter
* Sets minimum number of executions to form a cluster
* DBSCAN hyperparameter.
*
* Sets minimum number of executions to form a cluster.
*/
var MIN_EXEC_DBSCAN: Int = 2

/**
* DBSCAN hyperparameter
* Sets radius of search for algorithm
* DBSCAN hyperparameter.
*
* Sets radius of search for algorithm.
*/
var RADIUS_DBSCAN: Float = 5.0f
}
Expand Down