From 1fe51035912a3bb12fddd88f83a1298efcb66175 Mon Sep 17 00:00:00 2001 From: Vladislav Frolov <50615459+Cheshiriks@users.noreply.github.com> Date: Mon, 23 Jan 2023 18:28:34 +0300 Subject: [PATCH] Test for actual warning in SAVE from the file (#366) * Support reading of actual warning in SAVE from the file What's done: * Added reading of actual warning from the file Co-authored-by: Kirill Gevorkyan <26010098+kgevorkyan@users.noreply.github.com> --- .../save-warnings-actual.sarif | 50 +++++++++++++++++++ .../save-warnings-expected.sarif | 50 +++++++++++++++++++ .../sarif-actual-expected/save.toml | 21 ++++++++ .../src/kotlin/EnumValueSnakeCaseTest.kt | 15 ++++++ .../save/core/integration/ClassicWarnTest.kt | 8 +++ .../save/plugin/warn/WarnPlugin.kt | 50 ++++++++++++------- 6 files changed, 175 insertions(+), 19 deletions(-) create mode 100644 examples/kotlin-diktat/sarif-actual-expected/save-warnings-actual.sarif create mode 100644 examples/kotlin-diktat/sarif-actual-expected/save-warnings-expected.sarif create mode 100644 examples/kotlin-diktat/sarif-actual-expected/save.toml create mode 100644 examples/kotlin-diktat/sarif-actual-expected/src/kotlin/EnumValueSnakeCaseTest.kt diff --git a/examples/kotlin-diktat/sarif-actual-expected/save-warnings-actual.sarif b/examples/kotlin-diktat/sarif-actual-expected/save-warnings-actual.sarif new file mode 100644 index 000000000..958f57dbe --- /dev/null +++ b/examples/kotlin-diktat/sarif-actual-expected/save-warnings-actual.sarif @@ -0,0 +1,50 @@ +{ + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "originalUriBaseIds": { + "%SRCROOT%": { + "uri": "file://D:/projects/" + } + }, + "results": [ + { + "level": "error", + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "src\\kotlin\\EnumValueSnakeCaseTest.kt", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startColumn": 5, + "startLine": 18 + } + } + } + ], + "message": { + "text": "[ENUM_VALUE] enum values should be in selected UPPER_CASE snake/PascalCase format: NAme_MYa_sayR_" + }, + "ruleId": "diktat-ruleset:identifier-naming" + } + ], + "tool": { + "driver": { + "downloadUri": "https://github.com/pinterest/ktlint/releases/tag/0.42.0", + "fullName": "ktlint", + "informationUri": "https://github.com/pinterest/ktlint/", + "language": "en", + "name": "ktlint", + "organization": "pinterest", + "rules": [ + ], + "semanticVersion": "0.42.0", + "version": "0.42.0" + } + } + } + ] +} diff --git a/examples/kotlin-diktat/sarif-actual-expected/save-warnings-expected.sarif b/examples/kotlin-diktat/sarif-actual-expected/save-warnings-expected.sarif new file mode 100644 index 000000000..958f57dbe --- /dev/null +++ b/examples/kotlin-diktat/sarif-actual-expected/save-warnings-expected.sarif @@ -0,0 +1,50 @@ +{ + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "originalUriBaseIds": { + "%SRCROOT%": { + "uri": "file://D:/projects/" + } + }, + "results": [ + { + "level": "error", + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "src\\kotlin\\EnumValueSnakeCaseTest.kt", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startColumn": 5, + "startLine": 18 + } + } + } + ], + "message": { + "text": "[ENUM_VALUE] enum values should be in selected UPPER_CASE snake/PascalCase format: NAme_MYa_sayR_" + }, + "ruleId": "diktat-ruleset:identifier-naming" + } + ], + "tool": { + "driver": { + "downloadUri": "https://github.com/pinterest/ktlint/releases/tag/0.42.0", + "fullName": "ktlint", + "informationUri": "https://github.com/pinterest/ktlint/", + "language": "en", + "name": "ktlint", + "organization": "pinterest", + "rules": [ + ], + "semanticVersion": "0.42.0", + "version": "0.42.0" + } + } + } + ] +} diff --git a/examples/kotlin-diktat/sarif-actual-expected/save.toml b/examples/kotlin-diktat/sarif-actual-expected/save.toml new file mode 100644 index 000000000..569d15b8b --- /dev/null +++ b/examples/kotlin-diktat/sarif-actual-expected/save.toml @@ -0,0 +1,21 @@ +[general] + tags = ["warn", "sarif"] + description = "Test warnings discovered by diKTat, reading expected warnings from SARIF file" + suiteName = "Only Warnings: with SARIF format" + +[warn] + # regular expression to detect tests + testNameRegex = ".*Test.kt" + expectedWarningsFormat = "SARIF" + expectedWarningsFileName = "save-warnings-expected.sarif" + actualWarningsFormat = "SARIF" + actualWarningsFileName = "save-warnings-actual.sarif" + # diktat warnings have the following format: filename.kt:1:15: [WARN_NAME] my warning + actualWarningsPattern="(\\w+\\..+):(\\d+):(\\d+): (\\[.*\\].*)" + fileNameCaptureGroupOut = 1 + lineCaptureGroupOut = 2 + columnCaptureGroupOut = 3 + messageCaptureGroupOut = 4 + exactWarningsMatch = false + warningTextHasColumn = true + warningTextHasLine = true \ No newline at end of file diff --git a/examples/kotlin-diktat/sarif-actual-expected/src/kotlin/EnumValueSnakeCaseTest.kt b/examples/kotlin-diktat/sarif-actual-expected/src/kotlin/EnumValueSnakeCaseTest.kt new file mode 100644 index 000000000..a7b1319b0 --- /dev/null +++ b/examples/kotlin-diktat/sarif-actual-expected/src/kotlin/EnumValueSnakeCaseTest.kt @@ -0,0 +1,15 @@ +package org.cqfn.diktat.test.resources.test.paragraph1.naming.enum_ + +// ;warn:3:1: [MISSING_KDOC_TOP_LEVEL] all public and internal top-level classes and functions should have Kdoc: EnumValueSnakeCaseTest (cannot be auto-corrected) +// ;warn:35: [WRONG_DECLARATIONS_ORDER] declarations of constants and enum members should be sorted alphabetically: enum entries order is incorrect +// ;warn:10:5: [ENUMS_SEPARATED] enum is incorrectly formatted: enums must end with semicolon +enum class EnumValueSnakeCaseTest { + // ;warn:$line+1:5: [ENUM_VALUE] enum values should be{{ in }}selected UPPER_CASE snake/PascalCase format: paSC_SAl_l + paSC_SAl_l, + + // ;warn:5: [ENUM_VALUE] enum values{{ should }}be in selected{{ UPPER_CASE }}snake/PascalCase format: PascAsl_f + PascAsl_f + // ;warn:$line-2:5: [ENUMS_SEPARATED] enum is incorrectly formatted: last enum entry must end with a comma + + // ;warn:1:9: {{.*}}[PACKAGE_NAME_INCORRECT_PREFIX] package name should start from company's domain: org.cqfn.save{{.*}} +} diff --git a/save-core/src/commonNonJsTest/kotlin/com/saveourtool/save/core/integration/ClassicWarnTest.kt b/save-core/src/commonNonJsTest/kotlin/com/saveourtool/save/core/integration/ClassicWarnTest.kt index a7648bde1..8c9db2d50 100644 --- a/save-core/src/commonNonJsTest/kotlin/com/saveourtool/save/core/integration/ClassicWarnTest.kt +++ b/save-core/src/commonNonJsTest/kotlin/com/saveourtool/save/core/integration/ClassicWarnTest.kt @@ -157,4 +157,12 @@ class ClassicWarnTest { 1 ) } + + @Test + fun `execute warn-plugin with actual and expected warnings from SARIF`() { + runTestsWithDiktat( + listOf("sarif-actual-expected"), + 1 + ) + } } diff --git a/save-plugins/warn-plugin/src/commonMain/kotlin/com/saveourtool/save/plugin/warn/WarnPlugin.kt b/save-plugins/warn-plugin/src/commonMain/kotlin/com/saveourtool/save/plugin/warn/WarnPlugin.kt index 7318d154d..bf557c3d9 100644 --- a/save-plugins/warn-plugin/src/commonMain/kotlin/com/saveourtool/save/plugin/warn/WarnPlugin.kt +++ b/save-plugins/warn-plugin/src/commonMain/kotlin/com/saveourtool/save/plugin/warn/WarnPlugin.kt @@ -20,6 +20,7 @@ import com.saveourtool.save.core.utils.ExecutionResult import com.saveourtool.save.core.utils.ProcessExecutionException import com.saveourtool.save.core.utils.ProcessTimeoutException import com.saveourtool.save.core.utils.SarifParsingException +import com.saveourtool.save.core.utils.calculatePathToSarifFile import com.saveourtool.save.core.utils.singleIsInstance import com.saveourtool.save.plugin.warn.sarif.toWarnings import com.saveourtool.save.plugin.warn.utils.CmdExecutorWarn @@ -34,7 +35,6 @@ import com.saveourtool.save.plugin.warn.utils.getLineNumber import io.github.detekt.sarif4k.SarifSchema210 import okio.FileSystem import okio.Path -import okio.Path.Companion.toPath import kotlin.random.Random import kotlinx.serialization.decodeFromString @@ -46,6 +46,7 @@ private typealias WarningMap = Map> * A plugin that runs an executable and verifies that it produces required warning messages. * @property testConfig */ +@Suppress("TooManyFunctions") class WarnPlugin( testConfig: TestConfig, testFiles: List, @@ -151,26 +152,20 @@ class WarnPlugin( warnMissingExpectedWarnings(warnPluginConfig, generalConfig, originalPaths) } - val result = try { - cmdExecutor.execCmdAndGetExecutionResults(redirectTo) - } catch (ex: ProcessTimeoutException) { - logWarn("The following tests took too long to run and were stopped: $originalPaths, timeout for single test: ${ex.timeoutMillis}") - return failTestResult(originalPaths, ex, execCmd) - } catch (ex: ProcessExecutionException) { - return failTestResult(originalPaths, ex, execCmd) - } - - val actualWarningsMap = try { - warnPluginConfig.actualWarningsFileName?.let { - val execResult = ExecutionResult( - result.code, - fs.readLines(warnPluginConfig.actualWarningsFileName.toPath()), - result.stderr - ) - collectActualWarningsWithLineNumbers(execResult, warnPluginConfig, workingDirectory) - } ?: collectActualWarningsWithLineNumbers(result, warnPluginConfig, workingDirectory) + val (actualWarningsMap, result) = try { + actualWarningsIfExistActualWarningsFile(warnPluginConfig, originalPaths, workingDirectory) } catch (ex: SarifParsingException) { return failTestResult(originalPaths, ex, execCmd) + } ?: run { + val result = try { + cmdExecutor.execCmdAndGetExecutionResults(redirectTo) + } catch (ex: ProcessTimeoutException) { + logWarn("The following tests took too long to run and were stopped: $originalPaths, timeout for single test: ${ex.timeoutMillis}") + return failTestResult(originalPaths, ex, execCmd) + } catch (ex: ProcessExecutionException) { + return failTestResult(originalPaths, ex, execCmd) + } + collectActualWarningsWithLineNumbers(result, warnPluginConfig, workingDirectory) to result } val resultsChecker = ResultsChecker( @@ -195,6 +190,23 @@ class WarnPlugin( }.asSequence() } + private fun actualWarningsIfExistActualWarningsFile( + warnPluginConfig: WarnPluginConfig, + originalPaths: List, + workingDirectory: Path, + ) = warnPluginConfig.actualWarningsFileName?.let { + val sarif = calculatePathToSarifFile( + sarifFileName = warnPluginConfig.actualWarningsFileName, + anchorTestFilePath = originalPaths.first() + ) + val execResult = ExecutionResult( + 0, + fs.readLines(sarif), + listOf("Warnings were obtained from SARIF file, no debug info is available") + ) + collectActualWarningsWithLineNumbers(execResult, warnPluginConfig, workingDirectory) to execResult + } + private fun createTestFiles(paths: List, warnPluginConfig: WarnPluginConfig): List { logDebug("Creating temp copy files of resources for WarnPlugin...") logTrace("Trying to create temp files for: $paths")