Introduce inter-process communication between utbot and utbot-spring-analyzer

.run/Debug All.run.xml

@@ -3,6 +3,7 @@
     <toRun name="Run IDEA" type="GradleRunConfiguration" />
     <toRun name="Listen for Engine Process" type="Remote" />
     <toRun name="Listen for Instrumented Process" type="Remote" />
+    <toRun name="Listen for Spring Analyzer Process" type="Remote" />
     <method v="2" />
   </configuration>
 </component>

.run/Debug Spring Analyzer Process.run.xml

@@ -0,0 +1,7 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Debug Spring Analyzer Process" type="CompoundRunConfigurationType">
+    <toRun name="Run IDEA" type="GradleRunConfiguration" />
+    <toRun name="Listen for Spring Analyzer Process" type="Remote" />
+    <method v="2" />
+  </configuration>
+</component>

.run/Listen for Spring Analyzer Process.run.xml

@@ -0,0 +1,15 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Listen for Spring Analyzer Process" type="Remote" folderName="Utility Configurations">
+    <option name="USE_SOCKET_TRANSPORT" value="true" />
+    <option name="SERVER_MODE" value="true" />
+    <option name="SHMEM_ADDRESS" />
+    <option name="HOST" value="localhost" />
+    <option name="PORT" value="5007" />
+    <option name="AUTO_RESTART" value="true" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5007" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
+    <method v="2" />
+  </configuration>
+</component>

gradle.properties

@@ -77,6 +77,7 @@ pytorchNativeVersion=1.9.1
 shadowJarVersion=7.1.2
 openblasVersion=0.3.10-1.5.4
 arpackNgVersion=3.7.0-1.5.4
+commonsLoggingVersion=1.2
 
 # configuration for build server
 #

settings.gradle.kts

@@ -71,3 +71,4 @@ if (goIde.split(",").contains(ideType)) {
 }
 
 include("utbot-spring-analyzer")
+include("utbot-spring-analyzer-model")

utbot-core/src/main/kotlin/org/utbot/common/StandardStreamUtil.kt

@@ -0,0 +1,21 @@
+package org.utbot.common
+
+import java.io.OutputStream
+import java.io.PrintStream
+
+fun silentlyCloseStandardStreams() {
+    // we should change out/err streams as not to spend time on user output
+    // and also because rd default logging system writes some initial values to stdout, polluting it as well
+    val tmpStream = PrintStream(object : OutputStream() {
+        override fun write(b: Int) {}
+    })
+    val prevOut = System.out
+    val prevError = System.err
+    System.setOut(tmpStream)
+    System.setErr(tmpStream)
+    // stdin/stderr should be closed as not to leave hanging descriptors
+    // and we cannot log any exceptions here as rd remote logging is still not configured
+    // so we pass any exceptions
+    silent { prevOut.close() }
+    silent { prevError.close() }
+}

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

@@ -296,6 +296,29 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
 
 // endregion
 
+// region spring analyzer process debug
+    /**
+     * The property is useful only for the IntelliJ IDEs.
+     * If the property is set in true the spring analyzer process opens a debug port.
+     * @see runInstrumentedProcessWithDebug
+     * @see org.utbot.intellij.plugin.process.EngineProcess
+     */
+    var runSpringAnalyzerProcessWithDebug by getBooleanProperty(false)
+
+    /**
+     * The spring analyzer process JDWP agent's port
+     * A debugger attaches to the port in order to debug the process.
+     */
+    var springAnalyzerProcessDebugPort by getIntProperty(5007)
+
+    /**
+     * Value of the suspend mode for the JDWP agent of the spring analyzer process.
+     * If the value is true, the spring analyzer process will suspend until a debugger attaches to it.
+     */
+    var suspendSpringAnalyzerProcessExecutionInDebugMode by getBooleanProperty(true)
+
+// endregion
+
 // region instrumented process debug
     /**
      * The instrumented process JDWP agent's port of the instrumented process.

utbot-framework-api/src/main/kotlin/org/utbot/framework/process/CommonProcessArgs.kt

@@ -0,0 +1,27 @@
+package org.utbot.framework.process
+
+import org.utbot.common.osSpecificJavaExecutable
+import org.utbot.framework.plugin.services.JdkInfoService
+import java.io.File
+import java.nio.file.Path
+import kotlin.io.path.pathString
+
+private val javaExecutablePathString: Path
+    get() = JdkInfoService.jdkInfoProvider.info.path.resolve("bin${File.separatorChar}${osSpecificJavaExecutable()}")
+
+fun obtainCommonProcessCommandLineArgs(
+    debugPort: Int,
+    runWithDebug: Boolean,
+    suspendExecutionInDebugMode: Boolean,
+): List<String> = buildList {
+    val suspendValue = if (suspendExecutionInDebugMode) "y" else "n"
+    val debugArgument = "-agentlib:jdwp=transport=dt_socket,server=n,suspend=${suspendValue},quiet=y,address=$debugPort"
+        .takeIf { runWithDebug }
+
+    add(javaExecutablePathString.pathString)
+    val javaVersionSpecificArgs = OpenModulesContainer.javaVersionSpecificArguments
+    if (javaVersionSpecificArgs.isNotEmpty()) {
+        addAll(javaVersionSpecificArgs)
+    }
+    debugArgument?.let { add(it) }
+}

utbot-framework-api/src/main/kotlin/org/utbot/framework/process/OpenModulesContainer.kt

@@ -11,6 +11,7 @@ object OpenModulesContainer {
     init {
         modulesContainer = buildList {
             openPackage("java.base", "sun.security.util")
+            openPackage("java.base", "sun.reflect.annotation")
             openPackage("java.base", "java.text")
             openPackage("java.base", "java.lang.invoke")
             openPackage("java.base", "jdk.internal.misc")

utbot-framework/build.gradle

@@ -1,3 +1,7 @@
+configurations {
+    fetchSpringAnalyzerJar
+}
+
 dependencies {
 
     api project(':utbot-fuzzers')
@@ -6,6 +10,8 @@ dependencies {
     api project(':utbot-framework-api')
     api project(':utbot-rd')
 
+    implementation project(':utbot-spring-analyzer-model')
+
     implementation group: 'com.jetbrains.rd', name: 'rd-framework', version: rdVersion
     implementation group: 'com.jetbrains.rd', name: 'rd-core', version: rdVersion
 
@@ -34,4 +40,12 @@ dependencies {
 
     implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-core', version: ksmtVersion
     implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-z3', version: ksmtVersion
+
+    fetchSpringAnalyzerJar project(path: ':utbot-spring-analyzer', configuration: 'springAnalyzerJar')
+}
+
+processResources {
+    from(configurations.fetchSpringAnalyzerJar) {
+        into "lib"
+    }
 }

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/Domain.kt

@@ -3,7 +3,6 @@ package org.utbot.framework.codegen.domain
 import org.utbot.framework.DEFAULT_EXECUTION_TIMEOUT_IN_INSTRUMENTED_PROCESS_MS
 import org.utbot.framework.codegen.domain.builtin.mockitoClassId
 import org.utbot.framework.codegen.domain.builtin.ongoingStubbingClassId
-import org.utbot.framework.codegen.domain.context.CgContext
 import org.utbot.framework.codegen.domain.models.CgClassId
 import org.utbot.framework.codegen.tree.argumentsClassId
 import org.utbot.framework.plugin.api.BuiltinClassId
@@ -751,7 +750,7 @@ sealed class TypeReplacementApproach {
      *
      * Currently used in Spring applications only.
      */
-    class ReplaceIfPossible(val configFqn: String) : TypeReplacementApproach()
+    class ReplaceIfPossible(val config: String) : TypeReplacementApproach()
 }
 
 abstract class DependencyInjectionFramework(

utbot-framework/src/main/kotlin/org/utbot/framework/process/EngineProcessMain.kt

@@ -34,6 +34,7 @@ import org.utbot.rd.RdSettingsContainerFactory
 import org.utbot.rd.findRdPort
 import org.utbot.rd.generated.settingsModel
 import org.utbot.rd.loggers.UtRdKLoggerFactory
+import org.utbot.rd.terminateOnException
 import org.utbot.sarif.RdSourceFindingStrategyFacade
 import org.utbot.sarif.SarifReport
 import org.utbot.summary.summarizeAll
@@ -79,6 +80,20 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
             File(it).toURI().toURL()
         }.toTypedArray())))
     }
+    watchdog.measureTimeForActiveCall(getSpringBeanQualifiedNames, "Getting Spring bean definitions") { params ->
+        val springAnalyzerProcess = SpringAnalyzerProcess.createBlocking()
+        val beans = springAnalyzerProcess.terminateOnException { _ ->
+            springAnalyzerProcess.getBeanQualifiedNames(
+                params.classpath.toList(),
+                params.config,
+                // TODO remove once spring-analyzer learns to find resources on its own, temporarily leaving it here for testing with hardcoded absolute paths
+                propertyFilesPaths = emptyList(),
+                xmlConfigurationPaths = emptyList()
+            ).toTypedArray()
+        }
+        springAnalyzerProcess.terminate()
+        beans
+    }
     watchdog.measureTimeForActiveCall(createTestGenerator, "Creating Test Generator") { params ->
         AnalyticsConfigureUtil.configureML()
         Instrumenter.adapter = RdInstrumenter(realProtocol.rdInstrumenterAdapter)

utbot-framework/src/main/kotlin/org/utbot/framework/process/SpringAnalyzerProcess.kt

@@ -0,0 +1,105 @@
+package org.utbot.framework.process
+
+import com.jetbrains.rd.util.lifetime.LifetimeDefinition
+import kotlinx.coroutines.runBlocking
+import mu.KotlinLogging
+import org.apache.commons.io.FileUtils
+import org.utbot.common.getPid
+import org.utbot.common.utBotTempDirectory
+import org.utbot.framework.UtSettings
+import org.utbot.rd.ProcessWithRdServer
+import org.utbot.rd.exceptions.InstantProcessDeathException
+import org.utbot.rd.generated.LoggerModel
+import org.utbot.rd.generated.loggerModel
+import org.utbot.rd.generated.synchronizationModel
+import org.utbot.rd.loggers.UtRdKLogger
+import org.utbot.rd.loggers.setupRdLogger
+import org.utbot.rd.onSchedulerBlocking
+import org.utbot.rd.rdPortArgument
+import org.utbot.rd.startBlocking
+import org.utbot.rd.startUtProcessWithRdServer
+import org.utbot.rd.terminateOnException
+import org.utbot.spring.process.generated.SpringAnalyzerParams
+import org.utbot.spring.process.generated.SpringAnalyzerProcessModel
+import org.utbot.spring.process.generated.springAnalyzerProcessModel
+import java.nio.file.Files
+
+class SpringAnalyzerProcessInstantDeathException :
+    InstantProcessDeathException(UtSettings.springAnalyzerProcessDebugPort, UtSettings.runSpringAnalyzerProcessWithDebug)
+
+private const val SPRING_ANALYZER_JAR_FILENAME = "utbot-spring-analyzer-shadow.jar"
+private val logger = KotlinLogging.logger {}
+private val rdLogger = UtRdKLogger(logger, "")
+
+class SpringAnalyzerProcess private constructor(
+    rdProcess: ProcessWithRdServer
+) : ProcessWithRdServer by rdProcess {
+
+    companion object {
+        private fun obtainProcessSpecificCommandLineArgs(port: Int): List<String> {
+            val jarFile =
+                Files.createDirectories(utBotTempDirectory.toFile().resolve("spring-analyzer").toPath())
+                    .toFile().resolve(SPRING_ANALYZER_JAR_FILENAME)
+            FileUtils.copyURLToFile(
+                this::class.java.classLoader.getResource("lib/$SPRING_ANALYZER_JAR_FILENAME"),
+                jarFile
+            )
+            return listOf(
+                "-Dorg.apache.commons.logging.LogFactory=org.utbot.rd.loggers.RDApacheCommonsLogFactory",
+                "-jar",
+                jarFile.path,
+                rdPortArgument(port)
+            )
+        }
+
+        fun createBlocking() = runBlocking { SpringAnalyzerProcess() }
+
+        suspend operator fun invoke(): SpringAnalyzerProcess = LifetimeDefinition().terminateOnException { lifetime ->
+            val rdProcess = startUtProcessWithRdServer(lifetime) { port ->
+                val cmd = obtainCommonProcessCommandLineArgs(
+                    debugPort = UtSettings.springAnalyzerProcessDebugPort,
+                    runWithDebug = UtSettings.runSpringAnalyzerProcessWithDebug,
+                    suspendExecutionInDebugMode = UtSettings.suspendSpringAnalyzerProcessExecutionInDebugMode,
+                ) + obtainProcessSpecificCommandLineArgs(port)
+                val process = ProcessBuilder(cmd)
+                    .directory(Files.createTempDirectory(utBotTempDirectory, "spring-analyzer").toFile())
+                    .start()
+
+                logger.info { "Spring Analyzer process started with PID = ${process.getPid}" }
+
+                if (!process.isAlive) throw SpringAnalyzerProcessInstantDeathException()
+
+                process
+            }
+            rdProcess.awaitProcessReady()
+            val proc = SpringAnalyzerProcess(rdProcess)
+            setupRdLogger(rdProcess, proc.loggerModel, rdLogger)
+            return proc
+        }
+    }
+
+    private val springAnalyzerModel: SpringAnalyzerProcessModel = onSchedulerBlocking { protocol.springAnalyzerProcessModel }
+    private val loggerModel: LoggerModel = onSchedulerBlocking { protocol.loggerModel }
+
+    init {
+        lifetime.onTermination {
+            protocol.synchronizationModel.stopProcess.fire(Unit)
+        }
+    }
+
+    fun getBeanQualifiedNames(
+        classpath: List<String>,
+        configuration: String,
+        propertyFilesPaths: List<String>,
+        xmlConfigurationPaths: List<String>
+    ): List<String> {
+        val params = SpringAnalyzerParams(
+            classpath.toTypedArray(),
+            configuration,
+            propertyFilesPaths.toTypedArray(),
+            xmlConfigurationPaths.toTypedArray()
+        )
+        val result = springAnalyzerModel.analyze.startBlocking(params)
+        return result.beanTypes.toList()
+    }
+}