Parametrized test generation in Uber tests #541

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -180,6 +180,8 @@ class UtBotSymbolicEngine(
 
     fun attachMockListener(mockListener: MockListener) = mocker.mockListenerController?.attach(mockListener)
 
+    fun detachMockListener(mockListener: MockListener) = mocker.mockListenerController?.detach(mockListener)
+
     private val statesForConcreteExecution: MutableList<ExecutionState> = mutableListOf()
 
     private val traverser = Traverser(

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/ForceMockListener.kt

@@ -2,6 +2,7 @@ package org.utbot.engine.util.mockListeners
 import org.utbot.engine.EngineController
 import org.utbot.engine.MockStrategy
 import org.utbot.engine.UtMockInfo
+import org.utbot.framework.plugin.api.TestCaseGenerator
 import org.utbot.framework.util.Conflict
 import org.utbot.framework.util.ConflictTriggers
 
@@ -18,4 +19,13 @@ class ForceMockListener(triggers: ConflictTriggers): MockListener(triggers) {
 
         triggers[Conflict.ForceMockHappened] = true
     }
+
+    companion object {
+        fun create(testCaseGenerator: TestCaseGenerator, conflictTriggers: ConflictTriggers) : ForceMockListener {
+            val listener = ForceMockListener(conflictTriggers)
+            testCaseGenerator.engineActions.add { engine -> engine.attachMockListener(listener) }
+
+            return listener
+        }
+    }
 }

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/ForceStaticMockListener.kt

@@ -6,6 +6,7 @@ import org.utbot.engine.UtMockInfo
 import org.utbot.engine.UtNewInstanceMockInfo
 import org.utbot.engine.UtStaticMethodMockInfo
 import org.utbot.engine.UtStaticObjectMockInfo
+import org.utbot.framework.plugin.api.TestCaseGenerator
 import org.utbot.framework.util.Conflict
 import org.utbot.framework.util.ConflictTriggers
 
@@ -26,4 +27,13 @@ class ForceStaticMockListener(triggers: ConflictTriggers): MockListener(triggers
             triggers[Conflict.ForceStaticMockHappened] = true
         }
     }
+
+    companion object {
+        fun create(testCaseGenerator: TestCaseGenerator, conflictTriggers: ConflictTriggers) : ForceStaticMockListener {
+            val listener = ForceStaticMockListener(conflictTriggers)
+            testCaseGenerator.engineActions.add { engine -> engine.attachMockListener(listener) }
+
+            return listener
+        }
+    }
 }

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/MockListener.kt

@@ -3,6 +3,7 @@ package org.utbot.engine.util.mockListeners
 import org.utbot.engine.EngineController
 import org.utbot.engine.MockStrategy
 import org.utbot.engine.UtMockInfo
+import org.utbot.framework.plugin.api.TestCaseGenerator
 import org.utbot.framework.util.ConflictTriggers
 
 /**
@@ -12,4 +13,8 @@ abstract class MockListener(
     val triggers: ConflictTriggers
 ) {
     abstract fun onShouldMock(controller: EngineController, strategy: MockStrategy, mockInfo: UtMockInfo)
+
+    fun detach(testCaseGenerator: TestCaseGenerator, listener: MockListener) {
+        testCaseGenerator.engineActions.add { engine -> engine.detachMockListener(listener) }
+    }
 }

utbot-framework/src/main/kotlin/org/utbot/engine/util/mockListeners/MockListenerController.kt

@@ -14,6 +14,10 @@ class MockListenerController(private val controller: EngineController) {
         listeners += listener
     }
 
+    fun detach(listener: MockListener) {
+        listeners -= listener
+    }
+
     fun onShouldMock(strategy: MockStrategy, mockInfo: UtMockInfo) {
         listeners.map { it.onShouldMock(controller, strategy, mockInfo) }
     }

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt

@@ -1217,11 +1217,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
     private val expectedResultVarName = "expectedResult"
     private val expectedErrorVarName = "expectedError"
 
-    fun createParameterizedTestMethod(testSet: UtMethodTestSet, dataProviderMethodName: String): CgTestMethod? {
-        if (testSet.executions.isEmpty()) {
-            return null
-        }
-
+    fun createParameterizedTestMethod(testSet: UtMethodTestSet, dataProviderMethodName: String): CgTestMethod {
         //TODO: orientation on generic execution may be misleading, but what is the alternative?
         //may be a heuristic to select a model with minimal number of internal nulls should be used
         val genericExecution = testSet.executions

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

@@ -52,7 +52,7 @@ internal class CgTestClassConstructor(val context: CgContext) :
                     cgDataProviderMethods.clear()
                     for (testSet in testSets) {
                         updateCurrentExecutable(testSet.method)
-                        val currentMethodUnderTestRegions = construct(testSet)
+                        val currentMethodUnderTestRegions = construct(testSet) ?: continue
                         val executableUnderTestCluster = CgExecutableUnderTestCluster(
                             "Test suites for executable $currentExecutable",
                             currentMethodUnderTestRegions
@@ -76,7 +76,11 @@ internal class CgTestClassConstructor(val context: CgContext) :
         }
     }
 
-    private fun construct(testSet: UtMethodTestSet): List<CgRegion<CgMethod>> {
+    private fun construct(testSet: UtMethodTestSet): List<CgRegion<CgMethod>>? {
+        if (testSet.executions.isEmpty()) {
+            return null
+        }
+
         val (methodUnderTest, executions, _, _, clustersInfo) = testSet
         val regions = mutableListOf<CgRegion<CgMethod>>()
         val requiredFields = mutableListOf<CgParameterDeclaration>()
@@ -107,17 +111,15 @@ internal class CgTestClassConstructor(val context: CgContext) :
                     val parameterizedTestMethod =
                         methodConstructor.createParameterizedTestMethod(testSet, dataProviderMethodName)
 
-                    if (parameterizedTestMethod != null) {
-                        requiredFields += parameterizedTestMethod.requiredFields
+                    requiredFields += parameterizedTestMethod.requiredFields
 
-                        cgDataProviderMethods +=
-                            methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
+                    cgDataProviderMethods +=
+                        methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
 
-                        regions += CgSimpleRegion(
-                            "Parameterized test for method ${methodUnderTest.displayName}",
-                            listOf(parameterizedTestMethod),
-                        )
-                    }
+                    regions += CgSimpleRegion(
+                        "Parameterized test for method ${methodUnderTest.displayName}",
+                        listOf(parameterizedTestMethod),
+                    )
                 }.onFailure { error -> processFailure(testSet, error) }
             }
         }

utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt

@@ -117,6 +117,8 @@ open class TestCaseGenerator(
         executionTimeEstimator: ExecutionTimeEstimator = ExecutionTimeEstimator(utBotGenerationTimeoutInMillis, 1)
     ): Flow<UtResult> {
         val engine = createSymbolicEngine(controller, method, mockStrategy, chosenClassesToMockAlways, executionTimeEstimator)
+        engineActions.map { engine.apply(it) }
+        engineActions.clear()
         return defaultTestFlow(engine, executionTimeEstimator.userTimeout)
     }
 

utbot-framework/src/test/kotlin/org/utbot/examples/TestSpecificTestCaseGenerator.kt

@@ -7,6 +7,8 @@ import org.utbot.common.runIgnoringCancellationException
 import org.utbot.engine.EngineController
 import org.utbot.engine.Mocker
 import org.utbot.engine.UtBotSymbolicEngine
+import org.utbot.engine.util.mockListeners.ForceMockListener
+import org.utbot.engine.util.mockListeners.ForceStaticMockListener
 import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.api.MockStrategyApi
 import org.utbot.framework.plugin.api.TestCaseGenerator
@@ -45,6 +47,9 @@ class TestSpecificTestCaseGenerator(
         val mockAlwaysDefaults = Mocker.javaDefaultClasses.mapTo(mutableSetOf()) { it.id }
         val defaultTimeEstimator = ExecutionTimeEstimator(UtSettings.utBotGenerationTimeoutInMillis, 1)
 
+        val forceMockListener = ForceMockListener.create(this, conflictTriggers)
+        val forceStaticMockListener = ForceStaticMockListener.create(this, conflictTriggers)
+
         runIgnoringCancellationException {
             runBlockingWithCancellationPredicate(isCanceled) {
                 super
@@ -58,6 +63,9 @@ class TestSpecificTestCaseGenerator(
             }
         }
 
+        forceMockListener.detach(this, forceMockListener)
+        forceStaticMockListener.detach(this, forceStaticMockListener)
+
         val minimizedExecutions = super.minimizeExecutions(executions)
         return UtMethodTestSet(method, minimizedExecutions, jimpleBody(method), errors)
     }

utbot-framework/src/test/kotlin/org/utbot/examples/TestUtil.kt

@@ -14,6 +14,8 @@ import org.utbot.framework.plugin.api.MockFramework
 import org.utbot.framework.plugin.api.MockStrategyApi
 import org.utbot.framework.plugin.api.MockStrategyApi.NO_MOCKS
 import org.utbot.framework.plugin.api.MockStrategyApi.OTHER_CLASSES
+import org.utbot.framework.util.Conflict
+import org.utbot.framework.util.ConflictTriggers
 
 
 data class TestFrameworkConfiguration(
@@ -28,6 +30,11 @@ data class TestFrameworkConfiguration(
     val runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.PASS,
     val enableTestsTimeout: Boolean = false // our tests should not fail due to timeout
 ) {
+    val isParametrizedAndMocked: Boolean
+        get() = parametrizedTestSource == ParametrizedTestSource.PARAMETRIZE &&
+                (mockStrategy != NO_MOCKS ||
+                        conflictTriggers[Conflict.ForceMockHappened] ?: false || conflictTriggers[Conflict.ForceStaticMockHappened] ?: false)
+
     val isDisabled: Boolean
         get() = run {
             // TODO Any? JIRA:1366
@@ -42,9 +49,6 @@ data class TestFrameworkConfiguration(
             // because otherwise the code generator will not create mocks even for mandatory to mock classes
             if (forceStaticMocking == ForceStaticMocking.FORCE && staticsMocking == NoStaticMocking) return true
 
-            // TODO find if mocks are used during the analysis JIRA:1418
-            if (parametrizedTestSource == ParametrizedTestSource.PARAMETRIZE) return true
-
             // junit4 doesn't support parametrized tests
             if (testFramework == Junit4 && parametrizedTestSource == ParametrizedTestSource.PARAMETRIZE) return true
 
@@ -55,6 +59,8 @@ data class TestFrameworkConfiguration(
         }
 }
 
+val conflictTriggers: ConflictTriggers = ConflictTriggers()
+
 val allTestFrameworkConfigurations: List<TestFrameworkConfiguration> = run {
     val possibleConfiguration = mutableListOf<TestFrameworkConfiguration>()
 

utbot-framework/src/test/kotlin/org/utbot/framework/codegen/TestCodeGeneratorPipeline.kt

@@ -7,6 +7,7 @@ import org.utbot.common.bracket
 import org.utbot.common.info
 import org.utbot.common.packageName
 import org.utbot.examples.TestFrameworkConfiguration
+import org.utbot.examples.conflictTriggers
 import org.utbot.framework.codegen.ExecutionStatus.SUCCESS
 import org.utbot.framework.codegen.model.CodeGenerator
 import org.utbot.framework.plugin.api.CodegenLanguage
@@ -71,22 +72,43 @@ class TestCodeGeneratorPipeline(private val testFrameworkConfiguration: TestFram
             val testSets = data as List<UtMethodTestSet>
 
             val codegenLanguage = testFrameworkConfiguration.codegenLanguage
+            val parametrizedTestSource = testFrameworkConfiguration.parametrizedTestSource
+            val isParametrizedAndMocked = testFrameworkConfiguration.isParametrizedAndMocked
 
             val testClass = callToCodeGenerator(testSets, classUnderTest)
 
+            // clear triggered flags from the current launch in order to get ready for the next possible run
+            conflictTriggers.clear()
+
             // actual number of the tests in the generated testClass
             val generatedMethodsCount = testClass
                 .lines()
                 .count {
                     val trimmedLine = it.trimStart()
-                    if (codegenLanguage == CodegenLanguage.JAVA) {
-                        trimmedLine.startsWith("public void")
-                    } else {
-                        trimmedLine.startsWith("fun ")
+                    val prefix = when (codegenLanguage) {
+                        CodegenLanguage.JAVA ->
+                            when (parametrizedTestSource) {
+                                ParametrizedTestSource.DO_NOT_PARAMETRIZE -> "public void "
+                                ParametrizedTestSource.PARAMETRIZE -> "public void parameterizedTestsFor"
+                            }
+
+                        CodegenLanguage.KOTLIN ->
+                            when (parametrizedTestSource) {
+                                ParametrizedTestSource.DO_NOT_PARAMETRIZE -> "fun "
+                                ParametrizedTestSource.PARAMETRIZE -> "fun parameterizedTestsFor"
+                            }
                     }
+                    trimmedLine.startsWith(prefix)
                 }
             // expected number of the tests in the generated testClass
-            val expectedNumberOfGeneratedMethods = testSets.sumOf { it.executions.size }
+            // if force mocking took place in parametrized test generation,
+            // we don't generate tests at all
+            val expectedNumberOfGeneratedMethods =
+                if (isParametrizedAndMocked) 0
+                else when (parametrizedTestSource) {
+                    ParametrizedTestSource.DO_NOT_PARAMETRIZE -> testSets.sumOf { it.executions.size }
+                    ParametrizedTestSource.PARAMETRIZE -> testSets.size
+                }
 
             // check for error in the generated file
             runCatching {
@@ -220,7 +242,12 @@ class TestCodeGeneratorPipeline(private val testFrameworkConfiguration: TestFram
         }
         val testClassCustomName = "${classUnderTest.java.simpleName}GeneratedTest"
 
-        return codeGenerator.generateAsString(testSets, testClassCustomName)
+        // if force mocking took place in parametrized test generation,
+        // we don't generate tests at all by passing empty list instead of test sets
+        return codeGenerator.generateAsString(
+            if (testFrameworkConfiguration.isParametrizedAndMocked) listOf() else testSets,
+            testClassCustomName
+        )
     }
 
     private fun checkPipelinesResults(classesPipelines: List<ClassPipeline>) {

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt

@@ -178,15 +178,11 @@ object UtTestsDialogProcessor {
                                     val mockFrameworkInstalled = model.mockFramework?.isInstalled ?: true
 
                                     if (!mockFrameworkInstalled) {
-                                         ForceMockListener(model.conflictTriggers).apply {
-                                             testCaseGenerator.engineActions.add { engine -> engine.attachMockListener(this) }
-                                         }
+                                        ForceMockListener.create(testCaseGenerator, model.conflictTriggers)
                                     }
 
                                     if (!model.staticsMocking.isConfigured) {
-                                        ForceStaticMockListener(model.conflictTriggers).apply {
-                                            testCaseGenerator.engineActions.add { engine -> engine.attachMockListener(this) }
-                                        }
+                                        ForceStaticMockListener.create(testCaseGenerator, model.conflictTriggers)
                                     }
 
                                     val notEmptyCases = withUtContext(context) {