Skip to content

Commit f833be0

Browse files
egiptipavelegiptipavel
authored
Go. Use coverage to stop fuzzing and speed up fuzzing (#1895)
* Stop fuzzing for function if coverage is 100% * Remove collecting comments in function modifier * Run all functions under test in one process * Increase timeout of end of worker execution
1 parent 9229a56 commit f833be0

File tree

8 files changed

+256
-235
lines changed

8 files changed

+256
-235
lines changed

‎utbot-go/src/main/kotlin/org/utbot/go/GoEngine.kt‎

Lines changed: 48 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -3,171 +3,75 @@ package org.utbot.go
33
import kotlinx.coroutines.flow.Flow
44
import kotlinx.coroutines.flow.flow
55
import mu.KotlinLogging
6-
import org.utbot.framework.plugin.api.TimeoutException
76
import org.utbot.fuzzing.BaseFeedback
87
import org.utbot.fuzzing.Control
98
import org.utbot.fuzzing.utils.Trie
10-
import org.utbot.go.api.*
11-
import org.utbot.go.imports.GoImportsResolver
12-
import org.utbot.go.logic.EachExecutionTimeoutsMillisConfig
13-
import org.utbot.go.util.executeCommandByNewProcessOrFailWithoutWaiting
9+
import org.utbot.go.api.GoUtExecutionResult
10+
import org.utbot.go.api.GoUtFunction
11+
import org.utbot.go.api.GoUtFuzzedFunction
12+
import org.utbot.go.api.GoUtPanicFailure
13+
import org.utbot.go.framework.api.go.GoPackage
1414
import org.utbot.go.worker.GoWorker
15-
import org.utbot.go.worker.GoWorkerCodeGenerationHelper
1615
import org.utbot.go.worker.convertRawExecutionResultToExecutionResult
17-
import java.io.File
18-
import java.io.InputStreamReader
19-
import java.net.ServerSocket
20-
import java.net.SocketTimeoutException
21-
import java.util.concurrent.TimeUnit
2216

2317
val logger = KotlinLogging.logger {}
2418

2519
class GoEngine(
20+
private val worker: GoWorker,
2621
private val functionUnderTest: GoUtFunction,
27-
private val sourceFile: GoUtFile,
22+
private val aliases: Map<GoPackage, String?>,
2823
private val intSize: Int,
29-
private val goExecutableAbsolutePath: String,
30-
private val eachExecutionTimeoutsMillisConfig: EachExecutionTimeoutsMillisConfig,
24+
private val eachExecutionTimeoutMillis: Long,
3125
private val timeoutExceededOrIsCanceled: () -> Boolean,
32-
private val timeoutMillis: Long = 10000
3326
) {
3427

3528
fun fuzzing(): Flow<Pair<GoUtFuzzedFunction, GoUtExecutionResult>> = flow {
3629
var attempts = 0
3730
val attemptsLimit = Int.MAX_VALUE
38-
ServerSocket(0).use { serverSocket ->
39-
var fileToExecute: File? = null
40-
var fileWithModifiedFunction: File? = null
41-
try {
42-
// creating files for worker
43-
val types = functionUnderTest.parameters.map { it.type }
44-
val imports = GoImportsResolver.resolveImportsBasedOnTypes(
45-
types,
46-
functionUnderTest.sourcePackage,
47-
GoWorkerCodeGenerationHelper.alwaysRequiredImports
48-
)
49-
fileToExecute = GoWorkerCodeGenerationHelper.createFileToExecute(
50-
sourceFile,
51-
functionUnderTest,
52-
eachExecutionTimeoutsMillisConfig,
53-
serverSocket.localPort,
54-
imports
55-
)
56-
fileWithModifiedFunction = GoWorkerCodeGenerationHelper.createFileWithModifiedFunction(
57-
sourceFile, functionUnderTest
58-
)
59-
60-
// starting worker process
61-
val testFunctionName = GoWorkerCodeGenerationHelper.workerTestFunctionName
62-
val command = listOf(
63-
goExecutableAbsolutePath, "test", "-run", testFunctionName
31+
if (functionUnderTest.parameters.isEmpty()) {
32+
worker.sendFuzzedParametersValues(functionUnderTest, emptyList(), emptyMap())
33+
val rawExecutionResult = worker.receiveRawExecutionResult()
34+
val executionResult = convertRawExecutionResultToExecutionResult(
35+
rawExecutionResult,
36+
functionUnderTest.resultTypes,
37+
intSize,
38+
eachExecutionTimeoutMillis,
39+
)
40+
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, emptyList())
41+
emit(fuzzedFunction to executionResult)
42+
} else {
43+
val notCoveredLines = (1..functionUnderTest.numberOfAllStatements).toMutableSet()
44+
runGoFuzzing(functionUnderTest, intSize) { description, values ->
45+
if (timeoutExceededOrIsCanceled() || notCoveredLines.isEmpty()) {
46+
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
47+
}
48+
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, values)
49+
worker.sendFuzzedParametersValues(functionUnderTest, values, aliases)
50+
val rawExecutionResult = worker.receiveRawExecutionResult()
51+
val executionResult = convertRawExecutionResultToExecutionResult(
52+
rawExecutionResult,
53+
functionUnderTest.resultTypes,
54+
intSize,
55+
eachExecutionTimeoutMillis,
6456
)
65-
val sourceFileDir = File(sourceFile.absoluteDirectoryPath)
66-
val processStartTime = System.currentTimeMillis()
67-
val process = executeCommandByNewProcessOrFailWithoutWaiting(command, sourceFileDir)
68-
69-
try {
70-
// connecting to worker
71-
logger.debug { "Trying to connect to worker" }
72-
val workerSocket = try {
73-
serverSocket.soTimeout = timeoutMillis.toInt()
74-
serverSocket.accept()
75-
} catch (e: SocketTimeoutException) {
76-
val processHasExited = process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)
77-
if (processHasExited) {
78-
val processOutput = InputStreamReader(process.inputStream).readText()
79-
throw TimeoutException("Timeout exceeded: Worker not connected. Process output: $processOutput")
80-
} else {
81-
process.destroy()
82-
}
83-
throw TimeoutException("Timeout exceeded: Worker not connected")
57+
if (executionResult.trace.isEmpty()) {
58+
logger.error { "Coverage is empty for [${functionUnderTest.name}] with $values}" }
59+
if (executionResult is GoUtPanicFailure) {
60+
logger.error { "Execution completed with panic: ${executionResult.panicValue}" }
8461
}
85-
val worker = GoWorker(workerSocket, functionUnderTest)
86-
logger.debug { "Worker connected - completed in ${System.currentTimeMillis() - processStartTime} ms" }
87-
88-
// fuzzing
89-
if (functionUnderTest.parameters.isEmpty()) {
90-
worker.sendFuzzedParametersValues(emptyList(), emptyMap())
91-
val rawExecutionResult = worker.receiveRawExecutionResult()
92-
val executionResult = convertRawExecutionResultToExecutionResult(
93-
rawExecutionResult,
94-
functionUnderTest.resultTypes,
95-
intSize,
96-
eachExecutionTimeoutsMillisConfig[functionUnderTest],
97-
)
98-
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, emptyList())
99-
emit(fuzzedFunction to executionResult)
100-
} else {
101-
val aliases = imports.filter { it.alias != null }.associate { it.goPackage to it.alias }
102-
runGoFuzzing(functionUnderTest, intSize) { description, values ->
103-
if (timeoutExceededOrIsCanceled()) {
104-
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
105-
}
106-
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, values)
107-
worker.sendFuzzedParametersValues(values, aliases)
108-
val rawExecutionResult = worker.receiveRawExecutionResult()
109-
val executionResult = convertRawExecutionResultToExecutionResult(
110-
rawExecutionResult,
111-
functionUnderTest.resultTypes,
112-
intSize,
113-
eachExecutionTimeoutsMillisConfig[functionUnderTest],
114-
)
115-
if (executionResult.trace.isEmpty()) {
116-
logger.error { "Coverage is empty for [${functionUnderTest.name}] with $values}" }
117-
if (executionResult is GoUtPanicFailure) {
118-
logger.error { "Execution completed with panic: ${executionResult.panicValue}" }
119-
}
120-
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
121-
}
122-
val trieNode = description.tracer.add(executionResult.trace.map { GoInstruction(it) })
123-
if (trieNode.count > 1) {
124-
if (++attempts >= attemptsLimit) {
125-
return@runGoFuzzing BaseFeedback(
126-
result = Trie.emptyNode(), control = Control.STOP
127-
)
128-
}
129-
return@runGoFuzzing BaseFeedback(result = trieNode, control = Control.CONTINUE)
130-
}
131-
emit(fuzzedFunction to executionResult)
132-
BaseFeedback(result = trieNode, control = Control.CONTINUE)
133-
}
134-
workerSocket.close()
135-
val processHasExited = process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)
136-
if (!processHasExited) {
137-
process.destroy()
138-
throw TimeoutException("Timeout exceeded: Worker didn't finish")
139-
}
140-
val exitCode = process.exitValue()
141-
if (exitCode != 0) {
142-
val processOutput = InputStreamReader(process.inputStream).readText()
143-
throw RuntimeException(
144-
StringBuilder()
145-
.append("Execution of ${"function [${functionUnderTest.name}] from $sourceFile"} in child process failed with non-zero exit code = $exitCode: ")
146-
.appendLine()
147-
.append(processOutput).toString()
148-
)
149-
}
150-
}
151-
} catch (e: Exception) {
152-
val processHasExited = process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)
153-
if (!processHasExited) {
154-
process.destroy()
155-
throw TimeoutException("Timeout exceeded: Worker didn't finish")
156-
}
157-
val exitCode = process.exitValue()
158-
if (exitCode != 0) {
159-
val processOutput = InputStreamReader(process.inputStream).readText()
160-
throw RuntimeException(
161-
StringBuilder()
162-
.append("Execution of ${"function [${functionUnderTest.name}] from $sourceFile"} in child process failed with non-zero exit code = $exitCode: ")
163-
.appendLine()
164-
.append(processOutput).toString()
165-
)
62+
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
63+
}
64+
val trieNode = description.tracer.add(executionResult.trace.map { GoInstruction(it) })
65+
if (trieNode.count > 1) {
66+
if (++attempts >= attemptsLimit) {
67+
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
16668
}
69+
return@runGoFuzzing BaseFeedback(result = trieNode, control = Control.CONTINUE)
70+
}
71+
if (notCoveredLines.removeAll(executionResult.trace.toSet())) {
72+
emit(fuzzedFunction to executionResult)
16773
}
168-
} finally {
169-
fileToExecute?.delete()
170-
fileWithModifiedFunction?.delete()
74+
BaseFeedback(result = trieNode, control = Control.CONTINUE)
17175
}
17276
}
17377
}

‎utbot-go/src/main/kotlin/org/utbot/go/logic/AbstractGoUtTestsGenerationController.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ abstract class AbstractGoUtTestsGenerationController {
2323
val numOfFunctions = analysisResults.values
2424
.map { it.functions.size }
2525
.reduce { acc, numOfFunctions -> acc + numOfFunctions }
26-
val functionTimeoutStepMillis = testsGenerationConfig.allExecutionTimeoutsMillisConfig / numOfFunctions
26+
val functionTimeoutStepMillis = testsGenerationConfig.allFunctionExecutionTimeoutMillis / numOfFunctions
2727
var startTimeMillis = System.currentTimeMillis()
2828
val testCasesBySourceFiles = analysisResults.mapValues { (sourceFile, analysisResult) ->
2929
val functions = analysisResult.functions
@@ -33,7 +33,7 @@ abstract class AbstractGoUtTestsGenerationController {
3333
functions,
3434
intSize,
3535
testsGenerationConfig.goExecutableAbsolutePath,
36-
testsGenerationConfig.eachExecutionTimeoutsMillisConfig
36+
testsGenerationConfig.eachFunctionExecutionTimeoutMillis
3737
) { index -> isCanceled() || System.currentTimeMillis() - (startTimeMillis + (index + 1) * functionTimeoutStepMillis) > 0 }
3838
.also {
3939
startTimeMillis += functionTimeoutStepMillis * functions.size

0 commit comments

Comments
 (0)