Test for actual warning in SAVE from the file (#366)

* Support reading of actual warning in SAVE from the file

What's done:
* Added reading of actual warning from the file

Co-authored-by: Kirill Gevorkyan <26010098+kgevorkyan@users.noreply.github.com>

examples/kotlin-diktat/sarif-actual-expected/save-warnings-actual.sarif

@@ -0,0 +1,50 @@
+{
+ "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [
+ {
+ "originalUriBaseIds": {
+ "%SRCROOT%": {
+ "uri": "file://D:/projects/"
+ }
+ },
+ "results": [
+ {
+ "level": "error",
+ "locations": [
+ {
+ "physicalLocation": {
+ "artifactLocation": {
+ "uri": "src\\kotlin\\EnumValueSnakeCaseTest.kt",
+ "uriBaseId": "%SRCROOT%"
+ },
+ "region": {
+ "startColumn": 5,
+ "startLine": 18
+ }
+ }
+ }
+ ],
+ "message": {
+ "text": "[ENUM_VALUE] enum values should be in selected UPPER_CASE snake/PascalCase format: NAme_MYa_sayR_"
+ },
+ "ruleId": "diktat-ruleset:identifier-naming"
+ }
+ ],
+ "tool": {
+ "driver": {
+ "downloadUri": "https://github.com/pinterest/ktlint/releases/tag/0.42.0",
+ "fullName": "ktlint",
+ "informationUri": "https://github.com/pinterest/ktlint/",
+ "language": "en",
+ "name": "ktlint",
+ "organization": "pinterest",
+ "rules": [
+ ],
+ "semanticVersion": "0.42.0",
+ "version": "0.42.0"
+ }
+ }
+ }
+ ]
+}

examples/kotlin-diktat/sarif-actual-expected/save-warnings-expected.sarif

@@ -0,0 +1,50 @@
+{
+ "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [
+ {
+ "originalUriBaseIds": {
+ "%SRCROOT%": {
+ "uri": "file://D:/projects/"
+ }
+ },
+ "results": [
+ {
+ "level": "error",
+ "locations": [
+ {
+ "physicalLocation": {
+ "artifactLocation": {
+ "uri": "src\\kotlin\\EnumValueSnakeCaseTest.kt",
+ "uriBaseId": "%SRCROOT%"
+ },
+ "region": {
+ "startColumn": 5,
+ "startLine": 18
+ }
+ }
+ }
+ ],
+ "message": {
+ "text": "[ENUM_VALUE] enum values should be in selected UPPER_CASE snake/PascalCase format: NAme_MYa_sayR_"
+ },
+ "ruleId": "diktat-ruleset:identifier-naming"
+ }
+ ],
+ "tool": {
+ "driver": {
+ "downloadUri": "https://github.com/pinterest/ktlint/releases/tag/0.42.0",
+ "fullName": "ktlint",
+ "informationUri": "https://github.com/pinterest/ktlint/",
+ "language": "en",
+ "name": "ktlint",
+ "organization": "pinterest",
+ "rules": [
+ ],
+ "semanticVersion": "0.42.0",
+ "version": "0.42.0"
+ }
+ }
+ }
+ ]
+}

examples/kotlin-diktat/sarif-actual-expected/save.toml

@@ -0,0 +1,21 @@
+[general]
+ tags = ["warn", "sarif"]
+ description = "Test warnings discovered by diKTat, reading expected warnings from SARIF file"
+ suiteName = "Only Warnings: with SARIF format"
+
+[warn]
+ # regular expression to detect tests
+ testNameRegex = ".*Test.kt"
+ expectedWarningsFormat = "SARIF"
+ expectedWarningsFileName = "save-warnings-expected.sarif"
+ actualWarningsFormat = "SARIF"
+ actualWarningsFileName = "save-warnings-actual.sarif"
+ # diktat warnings have the following format: filename.kt:1:15: [WARN_NAME] my warning
+ actualWarningsPattern="(\\w+\\..+):(\\d+):(\\d+): (\\[.*\\].*)"
+ fileNameCaptureGroupOut = 1
+ lineCaptureGroupOut = 2
+ columnCaptureGroupOut = 3
+ messageCaptureGroupOut = 4
+ exactWarningsMatch = false
+ warningTextHasColumn = true
+ warningTextHasLine = true

examples/kotlin-diktat/sarif-actual-expected/src/kotlin/EnumValueSnakeCaseTest.kt

@@ -0,0 +1,15 @@
+package org.cqfn.diktat.test.resources.test.paragraph1.naming.enum_
+
+// ;warn:3:1: [MISSING_KDOC_TOP_LEVEL] all public and internal top-level classes and functions should have Kdoc: EnumValueSnakeCaseTest (cannot be auto-corrected)
+// ;warn:35: [WRONG_DECLARATIONS_ORDER] declarations of constants and enum members should be sorted alphabetically: enum entries order is incorrect
+// ;warn:10:5: [ENUMS_SEPARATED] enum is incorrectly formatted: enums must end with semicolon
+enum class EnumValueSnakeCaseTest {
+ // ;warn:$line+1:5: [ENUM_VALUE] enum values should be{{ in }}selected UPPER_CASE snake/PascalCase format: paSC_SAl_l
+ paSC_SAl_l,
+
+ // ;warn:5: [ENUM_VALUE] enum values{{ should }}be in selected{{ UPPER_CASE }}snake/PascalCase format: PascAsl_f
+ PascAsl_f
+ // ;warn:$line-2:5: [ENUMS_SEPARATED] enum is incorrectly formatted: last enum entry must end with a comma
+
+ // ;warn:1:9: {{.*}}[PACKAGE_NAME_INCORRECT_PREFIX] package name should start from company's domain: org.cqfn.save{{.*}}
+}

save-core/src/commonNonJsTest/kotlin/com/saveourtool/save/core/integration/ClassicWarnTest.kt

@@ -157,4 +157,12 @@ class ClassicWarnTest {
  1
  )
  }
+
+ @Test
+ fun `execute warn-plugin with actual and expected warnings from SARIF`() {
+ runTestsWithDiktat(
+ listOf("sarif-actual-expected"),
+ 1
+ )
+ }
 }

save-plugins/warn-plugin/src/commonMain/kotlin/com/saveourtool/save/plugin/warn/WarnPlugin.kt

@@ -20,6 +20,7 @@ import com.saveourtool.save.core.utils.ExecutionResult
 import com.saveourtool.save.core.utils.ProcessExecutionException
 import com.saveourtool.save.core.utils.ProcessTimeoutException
 import com.saveourtool.save.core.utils.SarifParsingException
+import com.saveourtool.save.core.utils.calculatePathToSarifFile
 import com.saveourtool.save.core.utils.singleIsInstance
 import com.saveourtool.save.plugin.warn.sarif.toWarnings
 import com.saveourtool.save.plugin.warn.utils.CmdExecutorWarn
@@ -34,7 +35,6 @@ import com.saveourtool.save.plugin.warn.utils.getLineNumber
 import io.github.detekt.sarif4k.SarifSchema210
 import okio.FileSystem
 import okio.Path
-import okio.Path.Companion.toPath
 
 import kotlin.random.Random
 import kotlinx.serialization.decodeFromString
@@ -46,6 +46,7 @@ private typealias WarningMap = Map<String, List<Warning>>
  * A plugin that runs an executable and verifies that it produces required warning messages.
  * @property testConfig
  */
+@Suppress("TooManyFunctions")
 class WarnPlugin(
  testConfig: TestConfig,
  testFiles: List<String>,
@@ -151,26 +152,20 @@ class WarnPlugin(
  warnMissingExpectedWarnings(warnPluginConfig, generalConfig, originalPaths)
  }
 
- val result = try {
- cmdExecutor.execCmdAndGetExecutionResults(redirectTo)
- } catch (ex: ProcessTimeoutException) {
- logWarn("The following tests took too long to run and were stopped: $originalPaths, timeout for single test: ${ex.timeoutMillis}")
- return failTestResult(originalPaths, ex, execCmd)
- } catch (ex: ProcessExecutionException) {
- return failTestResult(originalPaths, ex, execCmd)
- }
-
- val actualWarningsMap = try {
- warnPluginConfig.actualWarningsFileName?.let {
- val execResult = ExecutionResult(
- result.code,
- fs.readLines(warnPluginConfig.actualWarningsFileName.toPath()),
- result.stderr
- )
- collectActualWarningsWithLineNumbers(execResult, warnPluginConfig, workingDirectory)
- } ?: collectActualWarningsWithLineNumbers(result, warnPluginConfig, workingDirectory)
+ val (actualWarningsMap, result) = try {
+ actualWarningsIfExistActualWarningsFile(warnPluginConfig, originalPaths, workingDirectory)
  } catch (ex: SarifParsingException) {
  return failTestResult(originalPaths, ex, execCmd)
+ } ?: run {
+ val result = try {
+ cmdExecutor.execCmdAndGetExecutionResults(redirectTo)
+ } catch (ex: ProcessTimeoutException) {
+ logWarn("The following tests took too long to run and were stopped: $originalPaths, timeout for single test: ${ex.timeoutMillis}")
+ return failTestResult(originalPaths, ex, execCmd)
+ } catch (ex: ProcessExecutionException) {
+ return failTestResult(originalPaths, ex, execCmd)
+ }
+ collectActualWarningsWithLineNumbers(result, warnPluginConfig, workingDirectory) to result
  }
 
  val resultsChecker = ResultsChecker(
@@ -195,6 +190,23 @@ class WarnPlugin(
  }.asSequence()
  }
 
+ private fun actualWarningsIfExistActualWarningsFile(
+ warnPluginConfig: WarnPluginConfig,
+ originalPaths: List<Path>,
+ workingDirectory: Path,
+ ) = warnPluginConfig.actualWarningsFileName?.let {
+ val sarif = calculatePathToSarifFile(
+ sarifFileName = warnPluginConfig.actualWarningsFileName,
+ anchorTestFilePath = originalPaths.first()
+ )
+ val execResult = ExecutionResult(
+ 0,
+ fs.readLines(sarif),
+ listOf("Warnings were obtained from SARIF file, no debug info is available")
+ )
+ collectActualWarningsWithLineNumbers(execResult, warnPluginConfig, workingDirectory) to execResult
+ }
+
  private fun createTestFiles(paths: List<Path>, warnPluginConfig: WarnPluginConfig): List<Path> {
  logDebug("Creating temp copy files of resources for WarnPlugin...")
  logTrace("Trying to create temp files for: $paths")