From b60564c283fb2c9d90c1dd742732c8b86fa100c0 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Mon, 31 Oct 2022 14:38:30 +0300 Subject: [PATCH 1/2] Unified error and exception clusters for all UtExecutions --- .../kotlin/org/utbot/summary/Summarization.kt | 79 ++++---------- .../kotlin/org/utbot/summary/TagGenerator.kt | 103 +++++++++++++----- 2 files changed, 100 insertions(+), 82 deletions(-) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt index d05c71a84e..588e10cf0e 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt @@ -23,15 +23,6 @@ import java.io.File import java.nio.file.Path import java.nio.file.Paths import mu.KotlinLogging -import org.utbot.framework.plugin.api.UtConcreteExecutionFailure -import org.utbot.framework.plugin.api.UtExecutionSuccess -import org.utbot.framework.plugin.api.UtExplicitlyThrownException -import org.utbot.framework.plugin.api.UtImplicitlyThrownException -import org.utbot.framework.plugin.api.UtOverflowFailure -import org.utbot.framework.plugin.api.UtSandboxFailure -import org.utbot.framework.plugin.api.UtStreamConsumingFailure -import org.utbot.framework.plugin.api.UtTimeoutException -import org.utbot.framework.plugin.api.util.humanReadableName import org.utbot.framework.plugin.api.util.jClass import org.utbot.fuzzer.FuzzedMethodDescription import org.utbot.fuzzer.FuzzedValue @@ -70,7 +61,10 @@ fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.g } fun UtMethodTestSet.summarize(searchDirectory: Path): UtMethodTestSet = - this.summarize(Instrumenter.adapter.computeSourceFileByClass(this.method.classId.jClass, searchDirectory), searchDirectory) + this.summarize( + Instrumenter.adapter.computeSourceFileByClass(this.method.classId.jClass, searchDirectory), + searchDirectory + ) class Summarization(val sourceFile: File?, val invokeDescriptions: List) { @@ -219,9 +213,8 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List { val clustersToReturn: MutableList = mutableListOf() - val executionsProducedByFuzzer = testSet.executions.filterIsInstance() - val successfulFuzzerExecutions = mutableListOf() - val unsuccessfulFuzzerExecutions = mutableListOf() + val testSetWithFuzzedExecutions = prepareTestSetWithFuzzedExecutions(testSet) + val executionsProducedByFuzzer = testSetWithFuzzedExecutions.executions as List if (executionsProducedByFuzzer.isNotEmpty()) { executionsProducedByFuzzer.forEach { utExecution -> @@ -236,44 +229,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List unsuccessfulFuzzerExecutions.add(utExecution) - is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution) - } } - if (successfulFuzzerExecutions.isNotEmpty()) { - val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet) - - clustersToReturn.add( - UtExecutionCluster( - UtClusterInfo(clusterHeader, null), - successfulFuzzerExecutions - ) - ) - } - - if (unsuccessfulFuzzerExecutions.isNotEmpty()) { - val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet) + val clusteredExecutions = groupFuzzedExecutions(testSetWithFuzzedExecutions) + clusteredExecutions.forEach { clustersToReturn.add( UtExecutionCluster( - UtClusterInfo(clusterHeader, null), - unsuccessfulFuzzerExecutions + UtClusterInfo(it.header), + it.executions ) ) } @@ -282,20 +252,6 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List() + + return UtMethodTestSet( + method = testSet.method, + executions = executions, + jimpleBody = testSet.jimpleBody, + errors = testSet.errors, + clustersInfo = testSet.clustersInfo + ) + } + /** Filter and copies executions with non-empty paths. */ private fun prepareTestSetWithEmptyPaths(testSet: UtMethodTestSet): UtMethodTestSet { val executions = diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt index 5bced41064..f97580cbae 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt @@ -2,18 +2,20 @@ package org.utbot.summary import org.utbot.framework.plugin.api.Step import org.utbot.framework.plugin.api.UtConcreteExecutionFailure -import org.utbot.framework.plugin.api.UtSymbolicExecution +import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionResult import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.UtExplicitlyThrownException import org.utbot.framework.plugin.api.UtImplicitlyThrownException -import org.utbot.framework.plugin.api.UtOverflowFailure import org.utbot.framework.plugin.api.UtMethodTestSet +import org.utbot.framework.plugin.api.UtOverflowFailure import org.utbot.framework.plugin.api.UtSandboxFailure import org.utbot.framework.plugin.api.UtStreamConsumingFailure +import org.utbot.framework.plugin.api.UtSymbolicExecution import org.utbot.framework.plugin.api.UtTimeoutException import org.utbot.framework.plugin.api.util.humanReadableName import org.utbot.framework.plugin.api.util.isCheckedException +import org.utbot.fuzzer.UtFuzzedExecution import org.utbot.summary.UtSummarySettings.MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING import org.utbot.summary.clustering.MatrixUniqueness import org.utbot.summary.clustering.SplitSteps @@ -29,7 +31,7 @@ class TagGenerator { if (clusteredExecutions.isNotEmpty()) { val listOfSplitSteps = clusteredExecutions.map { - val mUniqueness = MatrixUniqueness(it.executions) + val mUniqueness = MatrixUniqueness(it.executions as List) mUniqueness.splitSteps() } @@ -64,7 +66,7 @@ class TagGenerator { traceTagClusters.add( TraceTagCluster( cluster.header, - generateExecutionTags(cluster.executions, splitSteps), + generateExecutionTags(cluster.executions as List, splitSteps), TraceTagWithoutExecution( commonStepsInCluster.toList(), cluster.executions.first().result, @@ -88,17 +90,21 @@ private fun generateExecutionTags(executions: List, splitSt /** - * Splits executions into clusters - * By default there is 5 types of clusters: - * Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow - * These are split by the type of execution result + * Splits executions with empty paths into clusters. + * + * By default, there is 5 types of clusters: + * - Success + * - UnexpectedFail + * - ExpectedCheckedThrow + * - ExpectedUncheckedThrow + * - UnexpectedUncheckedThrow * * @return clustered executions */ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() val clusters = mutableListOf() - val commentPrefix = "OTHER:" + val commentPrefix = "OTHER:" val commentPostfix = "for method ${testSet.method.humanReadableName}" val grouped = methodExecutions.groupBy { it.result.clusterKind() } @@ -106,23 +112,56 @@ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS } - .map { (suffixId, group) -> - FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group) - } + clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix) + return clusters +} + +/** + * Splits fuzzed executions into clusters. + * + * By default, there is 5 types of clusters: + * - Success + * - UnexpectedFail + * - ExpectedCheckedThrow + * - ExpectedUncheckedThrow + * - UnexpectedUncheckedThrow + * + * @return clustered executions + */ +fun groupFuzzedExecutions(testSet: UtMethodTestSet): List { + val methodExecutions = testSet.executions.filterIsInstance() + val clusters = mutableListOf() + val commentPrefix = "FUZZER:" + val commentPostfix = "for method ${testSet.method.humanReadableName}" + + val grouped = methodExecutions.groupBy { it.result.clusterKind() } + + val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList() + if (successfulExecutions.isNotEmpty()) { + clusters += SuccessfulExecutionCluster( + "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix", + successfulExecutions.toList() + ) + } + + clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix) return clusters } /** - * Splits executions produced by symbolic execution engine into clusters - * By default there is 5 types of clusters: - * Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow - * These are split by the type of execution result + * Splits symbolic executions produced by symbolic execution engine into clusters. + * + * By default, there is 5 types of clusters: + * - Success + * - UnexpectedFail + * - ExpectedCheckedThrow + * - ExpectedUncheckedThrow + * - UnexpectedUncheckedThrow * * If Success cluster has more than MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING execution * then clustering algorithm splits those into more clusters @@ -132,7 +171,7 @@ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() val clusters = mutableListOf() - val commentPrefix = "SYMBOLIC EXECUTION:" + val commentPrefix = "SYMBOLIC EXECUTION:" val commentPostfix = "for method ${testSet.method.humanReadableName}" val grouped = methodExecutions.groupBy { it.result.clusterKind() } @@ -161,11 +200,21 @@ private fun toClusterExecutions(testSet: UtMethodTestSet): List>, + commentPrefix: String, + commentPostfix: String +): List { + val clusters = grouped .filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS } .map { (suffixId, group) -> - FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group) - } + FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group) + } + return clusters } @@ -197,18 +246,18 @@ private fun UtExecutionResult.clusterKind() = when (this) { /** * Structure used to represent execution cluster with header */ -sealed class ExecutionCluster(var header: String, val executions: List) +sealed class ExecutionCluster(var header: String, val executions: List) /** * Represents successful execution cluster */ -private class SuccessfulExecutionCluster(header: String, executions: List) : +private class SuccessfulExecutionCluster(header: String, executions: List) : ExecutionCluster(header, executions) /** * Represents failed execution cluster */ -private class FailedExecutionCluster(header: String, executions: List) : +private class FailedExecutionCluster(header: String, executions: List) : ExecutionCluster(header, executions) /** From 171bc1d1287d8144174495bf645cd60100c32ea6 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Mon, 31 Oct 2022 15:21:31 +0300 Subject: [PATCH 2/2] Fixed JavaDocs --- .../kotlin/org/utbot/summary/TagGenerator.kt | 33 ++++--------------- .../org/utbot/summary/UtSummarySettings.kt | 22 +++++++------ 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt index f97580cbae..683a3d6e64 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt @@ -92,14 +92,7 @@ private fun generateExecutionTags(executions: List, splitSt /** * Splits executions with empty paths into clusters. * - * By default, there is 5 types of clusters: - * - Success - * - UnexpectedFail - * - ExpectedCheckedThrow - * - ExpectedUncheckedThrow - * - UnexpectedUncheckedThrow - * - * @return clustered executions + * @return clustered executions. */ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() @@ -124,14 +117,7 @@ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() @@ -154,19 +140,12 @@ fun groupFuzzedExecutions(testSet: UtMethodTestSet): List { } /** - * Splits symbolic executions produced by symbolic execution engine into clusters. - * - * By default, there is 5 types of clusters: - * - Success - * - UnexpectedFail - * - ExpectedCheckedThrow - * - ExpectedUncheckedThrow - * - UnexpectedUncheckedThrow + * Splits symbolic executions into clusters. * - * If Success cluster has more than MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING execution - * then clustering algorithm splits those into more clusters + * If Success cluster has more than [MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING] execution + * then clustering algorithm splits those into more clusters. * - * @return clustered executions + * @return clustered executions. */ private fun toClusterExecutions(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/UtSummarySettings.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/UtSummarySettings.kt index 3e36d3d72c..b27e6e04bb 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/UtSummarySettings.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/UtSummarySettings.kt @@ -2,27 +2,27 @@ package org.utbot.summary object UtSummarySettings { /** - * If True test comments will be generated + * If True test comments will be generated. */ var GENERATE_COMMENTS = true /** - * If True cluster comments will be generated + * If True cluster comments will be generated. */ var GENERATE_CLUSTER_COMMENTS = true /** - * If True names for tests will be generated + * If True names for tests will be generated. */ var GENERATE_NAMES = true /** - * If True display names for tests will be generated + * If True display names for tests will be generated. */ var GENERATE_DISPLAY_NAMES = true /** - * generate display name in from -> to style + * If True display name in from -> to style will be generated. */ var GENERATE_DISPLAYNAME_FROM_TO_STYLE = true @@ -34,19 +34,21 @@ object UtSummarySettings { /** * Sets minimum number of successful execution - * for applying the clustering algorithm + * for applying the clustering algorithm. */ const val MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING: Int = 4 /** - * DBSCAN hyperparameter - * Sets minimum number of executions to form a cluster + * DBSCAN hyperparameter. + * + * Sets minimum number of executions to form a cluster. */ var MIN_EXEC_DBSCAN: Int = 2 /** - * DBSCAN hyperparameter - * Sets radius of search for algorithm + * DBSCAN hyperparameter. + * + * Sets radius of search for algorithm. */ var RADIUS_DBSCAN: Float = 5.0f }