Skip to content

Go. Use coverage to stop fuzzing and speed up fuzzing #1895

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 6, 2023
Merged

Go. Use coverage to stop fuzzing and speed up fuzzing #1895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7014def
Stop fuzzing for function if coverage is 100%
egiptipavel Feb 28, 2023
09c63b5
Remove collecting comments in function modifier
egiptipavel Feb 28, 2023
63ab197
Run all functions under test in one process
egiptipavel Mar 1, 2023
35c7ac7
Increase timeout of end of worker execution
egiptipavel Mar 1, 2023
8cc7aee
Merge branch 'main' into egiptipavel/go-coverage-and-run-all-function…
egiptipavel Mar 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 48 additions & 144 deletions utbot-go/src/main/kotlin/org/utbot/go/GoEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,171 +3,75 @@ package org.utbot.go
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.flow
import mu.KotlinLogging
import org.utbot.framework.plugin.api.TimeoutException
import org.utbot.fuzzing.BaseFeedback
import org.utbot.fuzzing.Control
import org.utbot.fuzzing.utils.Trie
import org.utbot.go.api.*
import org.utbot.go.imports.GoImportsResolver
import org.utbot.go.logic.EachExecutionTimeoutsMillisConfig
import org.utbot.go.util.executeCommandByNewProcessOrFailWithoutWaiting
import org.utbot.go.api.GoUtExecutionResult
import org.utbot.go.api.GoUtFunction
import org.utbot.go.api.GoUtFuzzedFunction
import org.utbot.go.api.GoUtPanicFailure
import org.utbot.go.framework.api.go.GoPackage
import org.utbot.go.worker.GoWorker
import org.utbot.go.worker.GoWorkerCodeGenerationHelper
import org.utbot.go.worker.convertRawExecutionResultToExecutionResult
import java.io.File
import java.io.InputStreamReader
import java.net.ServerSocket
import java.net.SocketTimeoutException
import java.util.concurrent.TimeUnit

val logger = KotlinLogging.logger {}

class GoEngine(
private val worker: GoWorker,
private val functionUnderTest: GoUtFunction,
private val sourceFile: GoUtFile,
private val aliases: Map<GoPackage, String?>,
private val intSize: Int,
private val goExecutableAbsolutePath: String,
private val eachExecutionTimeoutsMillisConfig: EachExecutionTimeoutsMillisConfig,
private val eachExecutionTimeoutMillis: Long,
private val timeoutExceededOrIsCanceled: () -> Boolean,
private val timeoutMillis: Long = 10000
) {

fun fuzzing(): Flow<Pair<GoUtFuzzedFunction, GoUtExecutionResult>> = flow {
var attempts = 0
val attemptsLimit = Int.MAX_VALUE
ServerSocket(0).use { serverSocket ->
var fileToExecute: File? = null
var fileWithModifiedFunction: File? = null
try {
// creating files for worker
val types = functionUnderTest.parameters.map { it.type }
val imports = GoImportsResolver.resolveImportsBasedOnTypes(
types,
functionUnderTest.sourcePackage,
GoWorkerCodeGenerationHelper.alwaysRequiredImports
)
fileToExecute = GoWorkerCodeGenerationHelper.createFileToExecute(
sourceFile,
functionUnderTest,
eachExecutionTimeoutsMillisConfig,
serverSocket.localPort,
imports
)
fileWithModifiedFunction = GoWorkerCodeGenerationHelper.createFileWithModifiedFunction(
sourceFile, functionUnderTest
)

// starting worker process
val testFunctionName = GoWorkerCodeGenerationHelper.workerTestFunctionName
val command = listOf(
goExecutableAbsolutePath, "test", "-run", testFunctionName
if (functionUnderTest.parameters.isEmpty()) {
worker.sendFuzzedParametersValues(functionUnderTest, emptyList(), emptyMap())
val rawExecutionResult = worker.receiveRawExecutionResult()
val executionResult = convertRawExecutionResultToExecutionResult(
rawExecutionResult,
functionUnderTest.resultTypes,
intSize,
eachExecutionTimeoutMillis,
)
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, emptyList())
emit(fuzzedFunction to executionResult)
} else {
val notCoveredLines = (1..functionUnderTest.numberOfAllStatements).toMutableSet()
runGoFuzzing(functionUnderTest, intSize) { description, values ->
if (timeoutExceededOrIsCanceled() || notCoveredLines.isEmpty()) {
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
}
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, values)
worker.sendFuzzedParametersValues(functionUnderTest, values, aliases)
val rawExecutionResult = worker.receiveRawExecutionResult()
val executionResult = convertRawExecutionResultToExecutionResult(
rawExecutionResult,
functionUnderTest.resultTypes,
intSize,
eachExecutionTimeoutMillis,
)
val sourceFileDir = File(sourceFile.absoluteDirectoryPath)
val processStartTime = System.currentTimeMillis()
val process = executeCommandByNewProcessOrFailWithoutWaiting(command, sourceFileDir)

try {
// connecting to worker
logger.debug { "Trying to connect to worker" }
val workerSocket = try {
serverSocket.soTimeout = timeoutMillis.toInt()
serverSocket.accept()
} catch (e: SocketTimeoutException) {
val processHasExited = process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)
if (processHasExited) {
val processOutput = InputStreamReader(process.inputStream).readText()
throw TimeoutException("Timeout exceeded: Worker not connected. Process output: $processOutput")
} else {
process.destroy()
}
throw TimeoutException("Timeout exceeded: Worker not connected")
if (executionResult.trace.isEmpty()) {
logger.error { "Coverage is empty for [${functionUnderTest.name}] with $values}" }
if (executionResult is GoUtPanicFailure) {
logger.error { "Execution completed with panic: ${executionResult.panicValue}" }
}
val worker = GoWorker(workerSocket, functionUnderTest)
logger.debug { "Worker connected - completed in ${System.currentTimeMillis() - processStartTime} ms" }

// fuzzing
if (functionUnderTest.parameters.isEmpty()) {
worker.sendFuzzedParametersValues(emptyList(), emptyMap())
val rawExecutionResult = worker.receiveRawExecutionResult()
val executionResult = convertRawExecutionResultToExecutionResult(
rawExecutionResult,
functionUnderTest.resultTypes,
intSize,
eachExecutionTimeoutsMillisConfig[functionUnderTest],
)
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, emptyList())
emit(fuzzedFunction to executionResult)
} else {
val aliases = imports.filter { it.alias != null }.associate { it.goPackage to it.alias }
runGoFuzzing(functionUnderTest, intSize) { description, values ->
if (timeoutExceededOrIsCanceled()) {
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
}
val fuzzedFunction = GoUtFuzzedFunction(functionUnderTest, values)
worker.sendFuzzedParametersValues(values, aliases)
val rawExecutionResult = worker.receiveRawExecutionResult()
val executionResult = convertRawExecutionResultToExecutionResult(
rawExecutionResult,
functionUnderTest.resultTypes,
intSize,
eachExecutionTimeoutsMillisConfig[functionUnderTest],
)
if (executionResult.trace.isEmpty()) {
logger.error { "Coverage is empty for [${functionUnderTest.name}] with $values}" }
if (executionResult is GoUtPanicFailure) {
logger.error { "Execution completed with panic: ${executionResult.panicValue}" }
}
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
}
val trieNode = description.tracer.add(executionResult.trace.map { GoInstruction(it) })
if (trieNode.count > 1) {
if (++attempts >= attemptsLimit) {
return@runGoFuzzing BaseFeedback(
result = Trie.emptyNode(), control = Control.STOP
)
}
return@runGoFuzzing BaseFeedback(result = trieNode, control = Control.CONTINUE)
}
emit(fuzzedFunction to executionResult)
BaseFeedback(result = trieNode, control = Control.CONTINUE)
}
workerSocket.close()
val processHasExited = process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)
if (!processHasExited) {
process.destroy()
throw TimeoutException("Timeout exceeded: Worker didn't finish")
}
val exitCode = process.exitValue()
if (exitCode != 0) {
val processOutput = InputStreamReader(process.inputStream).readText()
throw RuntimeException(
StringBuilder()
.append("Execution of ${"function [${functionUnderTest.name}] from $sourceFile"} in child process failed with non-zero exit code = $exitCode: ")
.appendLine()
.append(processOutput).toString()
)
}
}
} catch (e: Exception) {
val processHasExited = process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)
if (!processHasExited) {
process.destroy()
throw TimeoutException("Timeout exceeded: Worker didn't finish")
}
val exitCode = process.exitValue()
if (exitCode != 0) {
val processOutput = InputStreamReader(process.inputStream).readText()
throw RuntimeException(
StringBuilder()
.append("Execution of ${"function [${functionUnderTest.name}] from $sourceFile"} in child process failed with non-zero exit code = $exitCode: ")
.appendLine()
.append(processOutput).toString()
)
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
}
val trieNode = description.tracer.add(executionResult.trace.map { GoInstruction(it) })
if (trieNode.count > 1) {
if (++attempts >= attemptsLimit) {
return@runGoFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP)
}
return@runGoFuzzing BaseFeedback(result = trieNode, control = Control.CONTINUE)
}
if (notCoveredLines.removeAll(executionResult.trace.toSet())) {
emit(fuzzedFunction to executionResult)
}
} finally {
fileToExecute?.delete()
fileWithModifiedFunction?.delete()
BaseFeedback(result = trieNode, control = Control.CONTINUE)
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions utbot-go/src/main/kotlin/org/utbot/go/logic/AbstractGoUtTestsGenerationController.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ abstract class AbstractGoUtTestsGenerationController {
val numOfFunctions = analysisResults.values
.map { it.functions.size }
.reduce { acc, numOfFunctions -> acc + numOfFunctions }
val functionTimeoutStepMillis = testsGenerationConfig.allExecutionTimeoutsMillisConfig / numOfFunctions
val functionTimeoutStepMillis = testsGenerationConfig.allFunctionExecutionTimeoutMillis / numOfFunctions
var startTimeMillis = System.currentTimeMillis()
val testCasesBySourceFiles = analysisResults.mapValues { (sourceFile, analysisResult) ->
val functions = analysisResult.functions
Expand All @@ -33,7 +33,7 @@ abstract class AbstractGoUtTestsGenerationController {
functions,
intSize,
testsGenerationConfig.goExecutableAbsolutePath,
testsGenerationConfig.eachExecutionTimeoutsMillisConfig
testsGenerationConfig.eachFunctionExecutionTimeoutMillis
) { index -> isCanceled() || System.currentTimeMillis() - (startTimeMillis + (index + 1) * functionTimeoutStepMillis) > 0 }
.also {
startTimeMillis += functionTimeoutStepMillis * functions.size
Expand Down
Loading