Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reordered the test clusters #991

Merged
merged 8 commits into from
Sep 30, 2022
Merged

Reordered the test clusters #991

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ff78ec2
Reordered the test clusters
amandelpie Sep 21, 2022
3c2cc86
Merge branch 'main' into amandelpie/feature-990
onewhl Sep 27, 2022
a4f4740
Merge branch 'main' into amandelpie/feature-990
amandelpie Sep 28, 2022
eda80c7
Renamed clusters
amandelpie Sep 28, 2022
fcc8b1b
Fixed tests
amandelpie Sep 28, 2022
7b26240
Merge branch 'main' into amandelpie/feature-990
amandelpie Sep 28, 2022
d5d76f8
Fixed tests
amandelpie Sep 28, 2022
db2f7f4
Merge remote-tracking branch 'origin/amandelpie/feature-990' into ama…
amandelpie Sep 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ class SummaryIntMathTest : SummaryTestCaseGeneratorTest(
)

val clusterInfo = listOf(
Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14)
Pair(UtClusterInfo("SYMBOLIC EXECUTION: SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14)
)

val method = IntMath::pow
Expand Down
6 changes: 3 additions & 3 deletions utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,10 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest(
)

val clusterInfo = listOf(
Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3),
Pair(UtClusterInfo("SYMBOLIC EXECUTION: SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3),
Pair(
UtClusterInfo(
"SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" +
"SYMBOLIC EXECUTION: SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" +
"Common steps:\n" +
"<pre>\n" +
"Tests execute conditions:\n" +
Expand All @@ -246,7 +246,7 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest(
"</pre>"
), 3
),
Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: ERROR SUITE for method ofDoubles(double[])", null), 1)
Pair(UtClusterInfo("SYMBOLIC EXECUTION: ERROR SUITE for method ofDoubles(double[])", null), 1)
)

summaryCheck(method, mockStrategy, coverage, summaryKeys, methodNames, displayNames, clusterInfo)
Expand Down
200 changes: 112 additions & 88 deletions utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,6 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
private val jimpleBodyAnalysis = ExecutionStructureAnalysis()

fun fillSummaries(testSet: UtMethodTestSet): List<UtExecutionCluster> {
val namesCounter = mutableMapOf<String, Int>()

if (testSet.executions.isEmpty()) {
logger.info {
"No execution traces found in test case " +
Expand All @@ -87,98 +85,29 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
return listOf(UtExecutionCluster(UtClusterInfo(), testSet.executions))
}

// init
val sootToAST = sootToAST(testSet)
val jimpleBody = testSet.jimpleBody
val updatedExecutions = mutableListOf<UtSymbolicExecution>()
val clustersToReturn = mutableListOf<UtExecutionCluster>()

// handles tests produced by fuzzing
val executionsProducedByFuzzer = testSet.executions.filterIsInstance<UtFuzzedExecution>()
val successfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
val unsuccessfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()

if (executionsProducedByFuzzer.isNotEmpty()) {
executionsProducedByFuzzer.forEach { utExecution ->

val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester())
val testMethodName = try {
nameSuggester.flatMap {
it.suggest(
utExecution.fuzzedMethodDescription as FuzzedMethodDescription,
utExecution.fuzzingValues as List<FuzzedValue>,
utExecution.result
)
}.firstOrNull()
} catch (t: Throwable) {
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur
null
}

utExecution.testMethodName = testMethodName?.testName
utExecution.displayName = testMethodName?.displayName

when (utExecution.result) {
is UtConcreteExecutionFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtExplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtImplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtOverflowFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtSandboxFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtTimeoutException -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution)
}
}

if (successfulFuzzerExecutions.isNotEmpty()) {
val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet)

clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(clusterHeader, null),
successfulFuzzerExecutions
)
)
}

if (unsuccessfulFuzzerExecutions.isNotEmpty()) {
val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet)

clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(clusterHeader, null),
unsuccessfulFuzzerExecutions
)
)
}
}

// handles tests produced by symbolic engine, but with empty paths
val testSetWithEmptyPaths = prepareTestSetWithEmptyPaths(testSet)

val executionsWithEmptyPaths = testSetWithEmptyPaths.executions

if (executionsWithEmptyPaths.isNotEmpty()) {
executionsWithEmptyPaths.forEach {
logger.info {
"The path for test ${it.testMethodName} " +
"for method ${testSet.method.classId.name} is empty and summaries could not be generated."
}
}

val clusteredExecutions = groupExecutionsWithEmptyPaths(testSetWithEmptyPaths)
clustersToReturn += generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(testSet)
clustersToReturn += generateSummariesForTestsProducedByFuzzer(testSet)
clustersToReturn += generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(testSet)

clusteredExecutions.forEach {
clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(it.header),
it.executions
)
)
}
}
return if (clustersToReturn.size > 0)
clustersToReturn
else
listOf(UtExecutionCluster(UtClusterInfo(), testSet.executions))
}

private fun generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(
testSet: UtMethodTestSet
): List<UtExecutionCluster> {
val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
val testSetWithNonEmptyPaths = prepareTestSetForByteCodeAnalysis(testSet)

val sootToAST = sootToAST(testSetWithNonEmptyPaths)
val jimpleBody = testSet.jimpleBody
val updatedExecutions = mutableListOf<UtSymbolicExecution>()
val namesCounter = mutableMapOf<String, Int>()

// analyze
if (jimpleBody != null && sootToAST != null) {
val methodUnderTest = jimpleBody.method
Expand Down Expand Up @@ -255,6 +184,101 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
return listOf(UtExecutionCluster(UtClusterInfo(), testSet.executions))
}

private fun generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(
testSet: UtMethodTestSet,
): List<UtExecutionCluster> {
val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
val testSetWithEmptyPaths = prepareTestSetWithEmptyPaths(testSet)

val executionsWithEmptyPaths = testSetWithEmptyPaths.executions

if (executionsWithEmptyPaths.isNotEmpty()) {
executionsWithEmptyPaths.forEach {
logger.info {
"The path for test ${it.testMethodName} " +
"for method ${testSet.method.classId.name} is empty and summaries could not be generated."
}
}

val clusteredExecutions = groupExecutionsWithEmptyPaths(testSetWithEmptyPaths)

clusteredExecutions.forEach {
clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(it.header),
it.executions
)
)
}
}
return clustersToReturn.toList()
}

private fun generateSummariesForTestsProducedByFuzzer(
testSet: UtMethodTestSet
): List<UtExecutionCluster> {
val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
val executionsProducedByFuzzer = testSet.executions.filterIsInstance<UtFuzzedExecution>()
val successfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
val unsuccessfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()

if (executionsProducedByFuzzer.isNotEmpty()) {
executionsProducedByFuzzer.forEach { utExecution ->

val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester())
val testMethodName = try {
nameSuggester.flatMap {
it.suggest(
utExecution.fuzzedMethodDescription as FuzzedMethodDescription,
utExecution.fuzzingValues as List<FuzzedValue>,
utExecution.result
)
}.firstOrNull()
} catch (t: Throwable) {
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur
null
}

utExecution.testMethodName = testMethodName?.testName
utExecution.displayName = testMethodName?.displayName

when (utExecution.result) {
is UtConcreteExecutionFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtExplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtImplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtOverflowFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtSandboxFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtTimeoutException -> unsuccessfulFuzzerExecutions.add(utExecution)
is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution)
}
}

if (successfulFuzzerExecutions.isNotEmpty()) {
val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet)

clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(clusterHeader, null),
successfulFuzzerExecutions
)
)
}

if (unsuccessfulFuzzerExecutions.isNotEmpty()) {
val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet)

clustersToReturn.add(
UtExecutionCluster(
UtClusterInfo(clusterHeader, null),
unsuccessfulFuzzerExecutions
)
)
}
}

return clustersToReturn.toList()
}

private fun buildFuzzerClusterHeaderForSuccessfulExecutions(testSet: UtMethodTestSet): String {
val commentPrefix = "FUZZER:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"
Expand Down
6 changes: 3 additions & 3 deletions utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ private fun generateExecutionTags(executions: List<UtSymbolicExecution>, splitSt
fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionCluster> {
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
val clusters = mutableListOf<ExecutionCluster>()
val commentPrefix = "CONCRETE EXECUTION ENGINE:"
val commentPrefix = "OTHER:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"

val grouped = methodExecutions.groupBy { it.result.clusterKind() }
Expand All @@ -118,7 +118,7 @@ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionClust
}

/**
* Splits executions into clusters
* Splits executions produced by symbolic execution engine into clusters
* By default there is 5 types of clusters:
* Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
* These are split by the type of execution result
Expand All @@ -131,7 +131,7 @@ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionClust
private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
val clusters = mutableListOf<ExecutionCluster>()
val commentPrefix = "SYMBOLIC EXECUTION ENGINE:"
val commentPrefix = "SYMBOLIC EXECUTION:"
val commentPostfix = "for method ${testSet.method.humanReadableName}"

val grouped = methodExecutions.groupBy { it.result.clusterKind() }
Expand Down