Skip to content

Commit b60564c

Browse files
amandelpieamandelpie
committed
Unified error and exception clusters for all UtExecutions
1 parent 145da7d commit b60564c

File tree

2 files changed

+100
-82
lines changed

2 files changed

+100
-82
lines changed

‎utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt

Lines changed: 24 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,6 @@ import java.io.File
2323
import java.nio.file.Path
2424
import java.nio.file.Paths
2525
import mu.KotlinLogging
26-
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
27-
import org.utbot.framework.plugin.api.UtExecutionSuccess
28-
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
29-
import org.utbot.framework.plugin.api.UtImplicitlyThrownException
30-
import org.utbot.framework.plugin.api.UtOverflowFailure
31-
import org.utbot.framework.plugin.api.UtSandboxFailure
32-
import org.utbot.framework.plugin.api.UtStreamConsumingFailure
33-
import org.utbot.framework.plugin.api.UtTimeoutException
34-
import org.utbot.framework.plugin.api.util.humanReadableName
3526
import org.utbot.framework.plugin.api.util.jClass
3627
import org.utbot.fuzzer.FuzzedMethodDescription
3728
import org.utbot.fuzzer.FuzzedValue
@@ -70,7 +61,10 @@ fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.g
7061
}
7162

7263
fun UtMethodTestSet.summarize(searchDirectory: Path): UtMethodTestSet =
73-
this.summarize(Instrumenter.adapter.computeSourceFileByClass(this.method.classId.jClass, searchDirectory), searchDirectory)
64+
this.summarize(
65+
Instrumenter.adapter.computeSourceFileByClass(this.method.classId.jClass, searchDirectory),
66+
searchDirectory
67+
)
7468

7569

7670
class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDescription>) {
@@ -219,9 +213,8 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
219213
testSet: UtMethodTestSet
220214
): List<UtExecutionCluster> {
221215
val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
222-
val executionsProducedByFuzzer = testSet.executions.filterIsInstance<UtFuzzedExecution>()
223-
val successfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
224-
val unsuccessfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
216+
val testSetWithFuzzedExecutions = prepareTestSetWithFuzzedExecutions(testSet)
217+
val executionsProducedByFuzzer = testSetWithFuzzedExecutions.executions as List<UtFuzzedExecution>
225218

226219
if (executionsProducedByFuzzer.isNotEmpty()) {
227220
executionsProducedByFuzzer.forEach { utExecution ->
@@ -236,44 +229,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
236229
)
237230
}.firstOrNull()
238231
} catch (t: Throwable) {
239-
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur
232+
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behaviour
240233
null
241234
}
242-
243235
utExecution.testMethodName = testMethodName?.testName
244236
utExecution.displayName = testMethodName?.displayName
245237
utExecution.summary = testMethodName?.javaDoc
246-
247-
when (utExecution.result) {
248-
is UtConcreteExecutionFailure,
249-
is UtExplicitlyThrownException,
250-
is UtImplicitlyThrownException,
251-
is UtOverflowFailure,
252-
is UtSandboxFailure,
253-
is UtTimeoutException,
254-
is UtStreamConsumingFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
255-
is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution)
256-
}
257238
}
258239

259-
if (successfulFuzzerExecutions.isNotEmpty()) {
260-
val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet)
261-
262-
clustersToReturn.add(
263-
UtExecutionCluster(
264-
UtClusterInfo(clusterHeader, null),
265-
successfulFuzzerExecutions
266-
)
267-
)
268-
}
269-
270-
if (unsuccessfulFuzzerExecutions.isNotEmpty()) {
271-
val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet)
240+
val clusteredExecutions = groupFuzzedExecutions(testSetWithFuzzedExecutions)
272241

242+
clusteredExecutions.forEach {
273243
clustersToReturn.add(
274244
UtExecutionCluster(
275-
UtClusterInfo(clusterHeader, null),
276-
unsuccessfulFuzzerExecutions
245+
UtClusterInfo(it.header),
246+
it.executions
277247
)
278248
)
279249
}
@@ -282,20 +252,6 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
282252
return clustersToReturn.toList()
283253
}
284254

285-
private fun buildFuzzerClusterHeaderForSuccessfulExecutions(testSet: UtMethodTestSet): String {
286-
val commentPrefix = "FUZZER:"
287-
val commentPostfix = "for method ${testSet.method.humanReadableName}"
288-
289-
return "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix"
290-
}
291-
292-
private fun buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet: UtMethodTestSet): String {
293-
val commentPrefix = "FUZZER:"
294-
val commentPostfix = "for method ${testSet.method.humanReadableName}"
295-
296-
return "$commentPrefix ${ExecutionGroup.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS} $commentPostfix"
297-
}
298-
299255
/** Filter and copies executions with non-empty paths. */
300256
private fun prepareTestSetForByteCodeAnalysis(testSet: UtMethodTestSet): UtMethodTestSet {
301257
val executions =
@@ -311,6 +267,19 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
311267
)
312268
}
313269

270+
/** Filter and copies fuzzed executions. */
271+
private fun prepareTestSetWithFuzzedExecutions(testSet: UtMethodTestSet): UtMethodTestSet {
272+
val executions = testSet.executions.filterIsInstance<UtFuzzedExecution>()
273+
274+
return UtMethodTestSet(
275+
method = testSet.method,
276+
executions = executions,
277+
jimpleBody = testSet.jimpleBody,
278+
errors = testSet.errors,
279+
clustersInfo = testSet.clustersInfo
280+
)
281+
}
282+
314283
/** Filter and copies executions with non-empty paths. */
315284
private fun prepareTestSetWithEmptyPaths(testSet: UtMethodTestSet): UtMethodTestSet {
316285
val executions =

‎utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt

Lines changed: 76 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,20 @@ package org.utbot.summary
22

33
import org.utbot.framework.plugin.api.Step
44
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
5-
import org.utbot.framework.plugin.api.UtSymbolicExecution
5+
import org.utbot.framework.plugin.api.UtExecution
66
import org.utbot.framework.plugin.api.UtExecutionResult
77
import org.utbot.framework.plugin.api.UtExecutionSuccess
88
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
99
import org.utbot.framework.plugin.api.UtImplicitlyThrownException
10-
import org.utbot.framework.plugin.api.UtOverflowFailure
1110
import org.utbot.framework.plugin.api.UtMethodTestSet
11+
import org.utbot.framework.plugin.api.UtOverflowFailure
1212
import org.utbot.framework.plugin.api.UtSandboxFailure
1313
import org.utbot.framework.plugin.api.UtStreamConsumingFailure
14+
import org.utbot.framework.plugin.api.UtSymbolicExecution
1415
import org.utbot.framework.plugin.api.UtTimeoutException
1516
import org.utbot.framework.plugin.api.util.humanReadableName
1617
import org.utbot.framework.plugin.api.util.isCheckedException
18+
import org.utbot.fuzzer.UtFuzzedExecution
1719
import org.utbot.summary.UtSummarySettings.MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING
1820
import org.utbot.summary.clustering.MatrixUniqueness
1921
import org.utbot.summary.clustering.SplitSteps
@@ -29,7 +31,7 @@ class TagGenerator {
2931

3032
if (clusteredExecutions.isNotEmpty()) {
3133
val listOfSplitSteps = clusteredExecutions.map {
32-
val mUniqueness = MatrixUniqueness(it.executions)
34+
val mUniqueness = MatrixUniqueness(it.executions as List<UtSymbolicExecution>)
3335
mUniqueness.splitSteps()
3436
}
3537

@@ -64,7 +66,7 @@ class TagGenerator {
6466
traceTagClusters.add(
6567
TraceTagCluster(
6668
cluster.header,
67-
generateExecutionTags(cluster.executions, splitSteps),
69+
generateExecutionTags(cluster.executions as List<UtSymbolicExecution>, splitSteps),
6870
TraceTagWithoutExecution(
6971
commonStepsInCluster.toList(),
7072
cluster.executions.first().result,
@@ -88,41 +90,78 @@ private fun generateExecutionTags(executions: List<UtSymbolicExecution>, splitSt
8890

8991

9092
/**
91-
* Splits executions into clusters
92-
* By default there is 5 types of clusters:
93-
* Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
94-
* These are split by the type of execution result
93+
* Splits executions with empty paths into clusters.
94+
*
95+
* By default, there is 5 types of clusters:
96+
* - Success
97+
* - UnexpectedFail
98+
* - ExpectedCheckedThrow
99+
* - ExpectedUncheckedThrow
100+
* - UnexpectedUncheckedThrow
95101
*
96102
* @return clustered executions
97103
*/
98104
fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionCluster> {
99105
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
100106
val clusters = mutableListOf<ExecutionCluster>()
101-
val commentPrefix = "OTHER:"
107+
val commentPrefix = "OTHER:"
102108
val commentPostfix = "for method ${testSet.method.humanReadableName}"
103109

104110
val grouped = methodExecutions.groupBy { it.result.clusterKind() }
105111

106112
val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
107113
if (successfulExecutions.isNotEmpty()) {
108114
clusters += SuccessfulExecutionCluster(
109-
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
110-
successfulExecutions.toList())
115+
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
116+
successfulExecutions.toList()
117+
)
111118
}
112119

113-
clusters += grouped
114-
.filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
115-
.map { (suffixId, group) ->
116-
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
117-
}
120+
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
121+
return clusters
122+
}
123+
124+
/**
125+
* Splits fuzzed executions into clusters.
126+
*
127+
* By default, there is 5 types of clusters:
128+
* - Success
129+
* - UnexpectedFail
130+
* - ExpectedCheckedThrow
131+
* - ExpectedUncheckedThrow
132+
* - UnexpectedUncheckedThrow
133+
*
134+
* @return clustered executions
135+
*/
136+
fun groupFuzzedExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
137+
val methodExecutions = testSet.executions.filterIsInstance<UtFuzzedExecution>()
138+
val clusters = mutableListOf<ExecutionCluster>()
139+
val commentPrefix = "FUZZER:"
140+
val commentPostfix = "for method ${testSet.method.humanReadableName}"
141+
142+
val grouped = methodExecutions.groupBy { it.result.clusterKind() }
143+
144+
val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
145+
if (successfulExecutions.isNotEmpty()) {
146+
clusters += SuccessfulExecutionCluster(
147+
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
148+
successfulExecutions.toList()
149+
)
150+
}
151+
152+
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
118153
return clusters
119154
}
120155

121156
/**
122-
* Splits executions produced by symbolic execution engine into clusters
123-
* By default there is 5 types of clusters:
124-
* Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
125-
* These are split by the type of execution result
157+
* Splits symbolic executions produced by symbolic execution engine into clusters.
158+
*
159+
* By default, there is 5 types of clusters:
160+
* - Success
161+
* - UnexpectedFail
162+
* - ExpectedCheckedThrow
163+
* - ExpectedUncheckedThrow
164+
* - UnexpectedUncheckedThrow
126165
*
127166
* If Success cluster has more than MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING execution
128167
* then clustering algorithm splits those into more clusters
@@ -132,7 +171,7 @@ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionClust
132171
private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
133172
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
134173
val clusters = mutableListOf<ExecutionCluster>()
135-
val commentPrefix = "SYMBOLIC EXECUTION:"
174+
val commentPrefix = "SYMBOLIC EXECUTION:"
136175
val commentPostfix = "for method ${testSet.method.humanReadableName}"
137176

138177
val grouped = methodExecutions.groupBy { it.result.clusterKind() }
@@ -161,11 +200,21 @@ private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster
161200
}
162201
}
163202

164-
clusters += grouped
203+
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
204+
return clusters
205+
}
206+
207+
private fun addClustersOfFailedExecutions(
208+
grouped: Map<ExecutionGroup, List<UtExecution>>,
209+
commentPrefix: String,
210+
commentPostfix: String
211+
): List<FailedExecutionCluster> {
212+
val clusters = grouped
165213
.filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
166214
.map { (suffixId, group) ->
167-
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
168-
}
215+
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
216+
}
217+
169218
return clusters
170219
}
171220

@@ -197,18 +246,18 @@ private fun UtExecutionResult.clusterKind() = when (this) {
197246
/**
198247
* Structure used to represent execution cluster with header
199248
*/
200-
sealed class ExecutionCluster(var header: String, val executions: List<UtSymbolicExecution>)
249+
sealed class ExecutionCluster(var header: String, val executions: List<UtExecution>)
201250

202251
/**
203252
* Represents successful execution cluster
204253
*/
205-
private class SuccessfulExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
254+
private class SuccessfulExecutionCluster(header: String, executions: List<UtExecution>) :
206255
ExecutionCluster(header, executions)
207256

208257
/**
209258
* Represents failed execution cluster
210259
*/
211-
private class FailedExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
260+
private class FailedExecutionCluster(header: String, executions: List<UtExecution>) :
212261
ExecutionCluster(header, executions)
213262

214263
/**

0 commit comments

Comments
 (0)