From 11711c8392c181628df4f5a06f2a349af860963f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Bia=C5=82y?= Date: Sun, 21 Jul 2024 20:13:41 +0200 Subject: [PATCH 1/3] improved integration tests a little bit --- .github/CODEOWNERS | 1 + .github/workflows/ci.yml | 2 -- integration-tests/downloadYamlConfigs.sh | 6 +++--- .../src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala | 7 ++++++- 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100755 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100755 index 000000000..501570769 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +.github/ @lbialy @tgodzik \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2a79746c..ad0b0d8e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,8 +61,6 @@ jobs: env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} PGP_SECRET: ${{ secrets.PGP_SECRET }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} documentation: needs: test diff --git a/integration-tests/downloadYamlConfigs.sh b/integration-tests/downloadYamlConfigs.sh index ab22f4fe9..9e2debeff 100755 --- a/integration-tests/downloadYamlConfigs.sh +++ b/integration-tests/downloadYamlConfigs.sh @@ -26,13 +26,13 @@ do done -mkdir ./test-suite/jvm/src/it/resources/yaml/configs/ -find ./repositories -name '*.yaml' -exec cp -prv '{}' './test-suite/jvm/src/it/resources/yaml/configs/' ';' +mkdir -p ./src/test/resources/yaml/configs/ +find ./repositories -name '*.yaml' -exec cp -prv '{}' './src/test/resources/yaml/configs/' ';' LIB_YAML_PATH="" # Set the path to libyaml # In downloaded repositories contains some invalid yaml, below instructions can remove this yaml -for f in ./test-suite/jvm/src/it/resources/yaml/configs/*.yaml; do +for f in ./src/test/resources/yaml/configs/*.yaml; do cat $f | $LIB_YAML_PATH >/dev/null # if libyaml return error exit code, these means, that yaml is invalid diff --git a/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala b/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala index 2f4c7341b..407903849 100644 --- a/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala +++ b/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala @@ -10,7 +10,12 @@ abstract class YamlRunnerSpec extends munit.FunSuite { val verbose = false val predicate: os.Path => Boolean = _ => true - val yamlDirPath = getClass.getResource(resourcePath) + val yamlDirPath = getClass.getResource(resourcePath) + if (yamlDirPath == null) { + throw new IllegalArgumentException( + s"Resource $resourcePath not found, run integrations-tests/downloadYamlConfigs.sh first!" + ) + } val yamlDir = new File(yamlDirPath.getPath) val yamlPaths: List[os.Path] = yamlDir.listFiles().map(os.Path(_)).filter(predicate).toList From 7703795ff73a413ef4e6aba2afb939b3b7555afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Bia=C5=82y?= Date: Sun, 21 Jul 2024 20:43:09 +0200 Subject: [PATCH 2/3] improved integration tests a little bit further --- .gitignore | 5 ++++- integration-tests/README.md | 8 +++++++- integration-tests/downloadYamlConfigs.sh | 3 ++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 4f05bd83a..84ac4e06e 100644 --- a/.gitignore +++ b/.gitignore @@ -38,10 +38,13 @@ target/ # test files *.tml *.yaml -tests/repositories/ +integration-tests/repositories/ #general *.class *.log generated-docs + +# java=17.0.9-graalce if you use sdkman +.sdkmanrc \ No newline at end of file diff --git a/integration-tests/README.md b/integration-tests/README.md index 73bed4001..a81b687e4 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -1,3 +1,9 @@ ### Test suite -Yaml test suite are used in integration testing - https://github.com/yaml/yaml-test-suite \ No newline at end of file +Yaml test suite are used in integration testing - https://github.com/yaml/yaml-test-suite + +#### How to run integration tests + +1. Figure out if `yamlpp-events` is installed on your system or install it if not, it's perl's yaml pp module +2. Run `downloadYamlConfigs.sh` script to download yaml test suites +3. `sbt integration/test` to run tests \ No newline at end of file diff --git a/integration-tests/downloadYamlConfigs.sh b/integration-tests/downloadYamlConfigs.sh index 9e2debeff..3c9f4971c 100755 --- a/integration-tests/downloadYamlConfigs.sh +++ b/integration-tests/downloadYamlConfigs.sh @@ -29,7 +29,8 @@ done mkdir -p ./src/test/resources/yaml/configs/ find ./repositories -name '*.yaml' -exec cp -prv '{}' './src/test/resources/yaml/configs/' ';' -LIB_YAML_PATH="" # Set the path to libyaml +# lbialy: I think the process used here is 'yamlpp-events' so `which yamlpp-events` should be used +LIB_YAML_PATH=`which yamlpp-events` # Set the path to libyaml # In downloaded repositories contains some invalid yaml, below instructions can remove this yaml for f in ./src/test/resources/yaml/configs/*.yaml; do From 3ff55a3bb4b1a1824a2135722e509e36c9bbfa1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Bia=C5=82y?= Date: Mon, 22 Jul 2024 15:16:32 +0200 Subject: [PATCH 3/3] added outYaml comparisons, discovered that our Presenter is rendering yaml differently, added passing ScalarStyle to nodes so that we render nodes as they were, unborked 6H3V this way, this is like constant cramp in my gray matter, yaml is horrible --- .github/workflows/ci.yml | 2 + .../main/scala/org/virtuslab/yaml/Node.scala | 40 +++++- .../org/virtuslab/yaml/YamlEncoder.scala | 2 +- .../internal/dump/present/PresenterImpl.scala | 26 +++- .../dump/serialize/SerializerImpl.scala | 8 +- .../yaml/internal/load/compose/Composer.scala | 3 +- .../yaml/traverse/NodeVisitorSuite.scala | 14 +- .../org/virtuslab/yaml/RunnerResult.scala | 5 + .../org/virtuslab/yaml/TestMlEntry.scala | 17 +++ .../scala/org/virtuslab/yaml/TestRunner.scala | 34 ++++- .../org/virtuslab/yaml/K8sConfigSpec.scala | 5 +- .../yaml/OfficialTestSuiteSpec.scala | 14 +- .../org/virtuslab/yaml/YamlRunnerSpec.scala | 129 ++++++++++++++---- 13 files changed, 243 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad0b0d8e8..f2a79746c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,8 @@ jobs: env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} PGP_SECRET: ${{ secrets.PGP_SECRET }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} documentation: needs: test diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/Node.scala b/core/shared/src/main/scala/org/virtuslab/yaml/Node.scala index 92118205a..99602360e 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/Node.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/Node.scala @@ -23,11 +23,45 @@ sealed trait Node { } object Node { - final case class ScalarNode private[yaml] (value: String, tag: Tag, pos: Option[Range] = None) - extends Node + + import org.virtuslab.yaml.internal.load.reader.token.{ScalarStyle => ParserScalarStyle} + + sealed abstract class ScalarStyle(indicator: Char) { + def toParserStyle: ParserScalarStyle = this match { + case ScalarStyle.Plain => ParserScalarStyle.Plain + case ScalarStyle.DoubleQuoted => ParserScalarStyle.DoubleQuoted + case ScalarStyle.SingleQuoted => ParserScalarStyle.SingleQuoted + case ScalarStyle.Folded => ParserScalarStyle.Folded + case ScalarStyle.Literal => ParserScalarStyle.Literal + } + } + object ScalarStyle { + + def fromParserStyle(style: ParserScalarStyle): ScalarStyle = style match { + case ParserScalarStyle.Plain => ScalarStyle.Plain + case ParserScalarStyle.DoubleQuoted => ScalarStyle.DoubleQuoted + case ParserScalarStyle.SingleQuoted => ScalarStyle.SingleQuoted + case ParserScalarStyle.Folded => ScalarStyle.Folded + case ParserScalarStyle.Literal => ScalarStyle.Literal + } + + case object Plain extends ScalarStyle(' ') + case object DoubleQuoted extends ScalarStyle('"') + case object SingleQuoted extends ScalarStyle('\'') + case object Folded extends ScalarStyle('>') + case object Literal extends ScalarStyle('|') + } + + final case class ScalarNode private[yaml] ( + value: String, + tag: Tag, + style: ScalarStyle, + pos: Option[Range] = None + ) extends Node object ScalarNode { - def apply(value: String): ScalarNode = new ScalarNode(value, Tag.resolveTag(value)) + def apply(value: String): ScalarNode = + new ScalarNode(value, style = ScalarStyle.Plain, tag = Tag.resolveTag(value)) def unapply(node: ScalarNode): Option[(String, Tag)] = Some((node.value, node.tag)) } diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala b/core/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala index a3860a010..f8baf2d25 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala @@ -48,7 +48,7 @@ object YamlEncoder extends YamlEncoderCrossCompanionCompat { implicit def forOption[T](implicit encoder: YamlEncoder[T]): YamlEncoder[Option[T]] = { case Some(t) => encoder.asNode(t) - case None => Node.ScalarNode("", Tag.nullTag) + case None => Node.ScalarNode("", Tag.nullTag, Node.ScalarStyle.Plain) } implicit def forSet[T](implicit encoder: YamlEncoder[T]): YamlEncoder[Set[T]] = (nodes) => diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/present/PresenterImpl.scala b/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/present/PresenterImpl.scala index 344718242..0fae6713c 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/present/PresenterImpl.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/present/PresenterImpl.scala @@ -8,6 +8,7 @@ import org.virtuslab.yaml.Tag import org.virtuslab.yaml.internal.load.parse.EventKind import org.virtuslab.yaml.internal.load.parse.EventKind._ import org.virtuslab.yaml.internal.load.parse.NodeEventMetadata +import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle object PresenterImpl extends Presenter { override def asString(events: Seq[EventKind]): String = { @@ -30,11 +31,18 @@ object PresenterImpl extends Presenter { insertSequencePadding() pushAndIncreaseIndent(SequenceStart()) parseSequence(tail) - case Scalar(value, _, NodeEventMetadata(_, tag)) => + case Scalar(value, style, NodeEventMetadata(_, tag)) => insertSequencePadding() // todo escape string using doublequotes if (tag.contains(Tag.nullTag)) sb.append("!!null") - else sb.append(value) + else + style match { + case ScalarStyle.Plain => sb.append(value) + case ScalarStyle.DoubleQuoted => sb.append(s"\"$value\"") + case ScalarStyle.SingleQuoted => sb.append(s"'$value'") + case ScalarStyle.Folded => sb.append(value) + case ScalarStyle.Literal => sb.append(value) + } sb.append(newline) tail case DocumentStart(_) => parseNode(tail) @@ -50,8 +58,8 @@ object PresenterImpl extends Presenter { case MappingEnd :: tail => popAndDecreaseIndent() tail - case Scalar(value, _, _) :: tail => - appendKey(value) + case Scalar(value, style, _) :: tail => + appendKey(value, style) val rest = parseNode(tail) parseMapping(rest) case _ => events @@ -69,9 +77,15 @@ object PresenterImpl extends Presenter { parseSequence(rest) } - def appendKey(value: String) = { + def appendKey(value: String, style: ScalarStyle) = { sb.append(" " * indent) - sb.append(value) + style match { + case ScalarStyle.Plain => sb.append(value) + case ScalarStyle.DoubleQuoted => sb.append(s"\"$value\"") + case ScalarStyle.SingleQuoted => sb.append(s"'$value'") + case ScalarStyle.Folded => sb.append(value) + case ScalarStyle.Literal => sb.append(value) + } sb.append(": ") } diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala b/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala index 8b3bdd167..a9797d72e 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala @@ -29,6 +29,12 @@ object SerializerImpl extends Serializer { } private def convertScalarNode(node: Node.ScalarNode): Seq[EventKind] = - Seq(Scalar(node.value, metadata = NodeEventMetadata(tag = node.tag))) + Seq( + Scalar( + node.value, + style = node.style.toParserStyle, + metadata = NodeEventMetadata(tag = node.tag) + ) + ) } diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala b/core/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala index 0d8df42b1..9acd32267 100644 --- a/core/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala +++ b/core/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala @@ -48,7 +48,8 @@ object ComposerImpl extends Composer { composeMappingNode(tail, anchor, aliases) case s: EventKind.Scalar => val tag: Tag = s.metadata.tag.getOrElse(Tag.resolveTag(s.value, Some(s.style))) - val node = Node.ScalarNode(s.value, tag, head.pos) + val node = + Node.ScalarNode(s.value, tag, Node.ScalarStyle.fromParserStyle(s.style), head.pos) s.metadata.anchor.foreach(anchor => aliases.put(anchor, node)) Right(Result(node, tail)) // todo #88 diff --git a/core/shared/src/test/scala/org/virtuslab/yaml/traverse/NodeVisitorSuite.scala b/core/shared/src/test/scala/org/virtuslab/yaml/traverse/NodeVisitorSuite.scala index 65db7e6b9..436946eba 100644 --- a/core/shared/src/test/scala/org/virtuslab/yaml/traverse/NodeVisitorSuite.scala +++ b/core/shared/src/test/scala/org/virtuslab/yaml/traverse/NodeVisitorSuite.scala @@ -27,7 +27,7 @@ class NodeVisitorSuite extends munit.FunSuite { val modifiedYaml = modifiedNode.asYaml val exptectedYaml = - s"""version: 3.9 + s"""version: "3.9" |services: | web: | build: . @@ -35,9 +35,9 @@ class NodeVisitorSuite extends munit.FunSuite { | - .:/code | - logvolume01:/var/log | ports: - | - 6000:6000 + | - "6000:6000" | redis: - | image: redis:alpine + | image: "redis:alpine" |""".stripMargin assertEquals(modifiedYaml, exptectedYaml) @@ -71,7 +71,7 @@ class NodeVisitorSuite extends munit.FunSuite { val modifiedYaml = modifiedNode.asYaml val exptectedYaml = - s"""version: 3.9 + s"""version: "3.9" |services: | web: | build: . @@ -79,9 +79,9 @@ class NodeVisitorSuite extends munit.FunSuite { | - .:/code | - logvolume01:/var/log | ports: - | - 5000:5000:6000 + | - "5000:5000:6000" | redis: - | image: redis:alpine:latest + | image: "redis:alpine:latest" |""".stripMargin assertEquals(modifiedYaml, exptectedYaml) @@ -115,7 +115,7 @@ class NodeVisitorSuite extends munit.FunSuite { val modifiedYaml = modifiedNode.asYaml val exptectedYaml = - s"""version: 3.9 + s"""version: "3.9" |services: | web: | build: . diff --git a/integration-tests/src/main/scala/org/virtuslab/yaml/RunnerResult.scala b/integration-tests/src/main/scala/org/virtuslab/yaml/RunnerResult.scala index b791736a2..51daaca58 100644 --- a/integration-tests/src/main/scala/org/virtuslab/yaml/RunnerResult.scala +++ b/integration-tests/src/main/scala/org/virtuslab/yaml/RunnerResult.scala @@ -20,6 +20,11 @@ object RunnerResult { else Error(eventsAsStr, error) } + case class InvalidOutYaml(eventsResult: RunnerResult, outYaml: String, expectedOutYaml: String) + extends RunnerResult { + override val isValid = false + } + case class Success(events: String) extends RunnerResult { override val isValid = true } diff --git a/integration-tests/src/main/scala/org/virtuslab/yaml/TestMlEntry.scala b/integration-tests/src/main/scala/org/virtuslab/yaml/TestMlEntry.scala index e3463e0b0..3307b300b 100644 --- a/integration-tests/src/main/scala/org/virtuslab/yaml/TestMlEntry.scala +++ b/integration-tests/src/main/scala/org/virtuslab/yaml/TestMlEntry.scala @@ -5,6 +5,7 @@ case class TestMlEntry( from: String, tags: String, inYaml: String, + outYaml: Option[String], seqEvent: String ) @@ -24,6 +25,21 @@ case object TestMlEntry { .head } + private def extractOutYaml(testMl: String): Option[String] = { + val patternOut = + raw"--- out-yaml(\(<\)|\(\+\)|\(<\+\)|)(([^\n]*\n+)+?)--- (in-json|error|emit-yaml|test-event)".r + + patternOut + .findAllIn(testMl) + .matchData + .map { m => + m.group(2) + } + .toList + .headOption + .map(_.strip()) + } + private def extractSeqEvent(testMl: String): String = { val patternEvent = raw"--- test-event(([^\n]*\n+)+).*".r @@ -42,6 +58,7 @@ case object TestMlEntry { from = "", tags = "", inYaml = extractInYaml(content), + outYaml = extractOutYaml(content), seqEvent = extractSeqEvent(content) ) } diff --git a/integration-tests/src/main/scala/org/virtuslab/yaml/TestRunner.scala b/integration-tests/src/main/scala/org/virtuslab/yaml/TestRunner.scala index 2b358c0f4..01a06fd14 100644 --- a/integration-tests/src/main/scala/org/virtuslab/yaml/TestRunner.scala +++ b/integration-tests/src/main/scala/org/virtuslab/yaml/TestRunner.scala @@ -16,7 +16,9 @@ import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle import org.virtuslab.yaml.internal.load.reader.Tokenizer trait TestRunner { + def testOutYaml: Boolean def inYaml: String + def outYaml: Option[String] = None def expectedEvents: String def run(): RunnerResult = { @@ -35,7 +37,28 @@ trait TestRunner { RunnerResult(acc.toList, expectedEvents, error) } } - loop() + + val result = loop() + + if (testOutYaml) { + // we don't compare output yaml if there is an error in event parsing + if ( + outYaml.isDefined && (result match { + case RunnerResult.Error(_, _) => false; case _ => true + }) + ) { + val roundtrip = inYaml.asNode match + case Left(error) => + println(s"Error while parsing inYaml:\n$error") + s"!!! ERROR: $error" + case Right(value) => value.asYaml + + if (roundtrip != outYaml.map(_.appended('\n')).get) + RunnerResult.InvalidOutYaml(result, roundtrip, outYaml.get) + else result + } else result + + } else result } } @@ -69,10 +92,11 @@ object TestRunnerUtils { case MappingEnd => "-MAP" case Alias(alias) => s"=ALI *$alias" case Scalar(value, style, data) => + val escapedValue = value.replace("\\", "\\\\").replace("\n", "\\n") style match { case ScalarStyle.Plain => s"=VAL${data.asString} :$value" case ScalarStyle.DoubleQuoted => s"""=VAL${data.asString} "$value""" - case ScalarStyle.SingleQuoted => s"=VAL${data.asString} '$value" + case ScalarStyle.SingleQuoted => s"=VAL${data.asString} '$escapedValue" case ScalarStyle.Folded => s"=VAL${data.asString} >$value" case ScalarStyle.Literal => s"=VAL${data.asString} |$value" } @@ -81,7 +105,8 @@ object TestRunnerUtils { } -case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path) extends TestRunner { +case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path, testOutYaml: Boolean) + extends TestRunner { override val inYaml = os.read(yamlPath) override val expectedEvents = os .proc(libYaml, yamlPath) @@ -97,9 +122,10 @@ case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path) extends TestRu } -case class YamlSuiteTestRunner(testYamlML: os.Path) extends TestRunner { +case class YamlSuiteTestRunner(testYamlML: os.Path, testOutYaml: Boolean) extends TestRunner { private val testMl = TestMlEntry.from(testYamlML) + override val outYaml = testMl.outYaml override val inYaml = testMl.inYaml override val expectedEvents = testMl.seqEvent diff --git a/integration-tests/src/test/scala/org/virtuslab/yaml/K8sConfigSpec.scala b/integration-tests/src/test/scala/org/virtuslab/yaml/K8sConfigSpec.scala index 3d7fd3585..f5fce3194 100644 --- a/integration-tests/src/test/scala/org/virtuslab/yaml/K8sConfigSpec.scala +++ b/integration-tests/src/test/scala/org/virtuslab/yaml/K8sConfigSpec.scala @@ -4,7 +4,8 @@ class K8sConfigSpec extends YamlRunnerSpec { val libYamlPath = os.Path(System.getenv("LIB_YAML_PATH")) - override def resourcePath: String = "/yaml/configs" - def createTestRunner(yamlPath: os.Path): TestRunner = K8sYamlTestRunner(yamlPath, libYamlPath) + override def resourcePath: String = "/yaml/configs" + def createTestRunner(yamlPath: os.Path): TestRunner = + K8sYamlTestRunner(yamlPath, libYamlPath, testOutYaml) } diff --git a/integration-tests/src/test/scala/org/virtuslab/yaml/OfficialTestSuiteSpec.scala b/integration-tests/src/test/scala/org/virtuslab/yaml/OfficialTestSuiteSpec.scala index 1aa3635b8..b7ebc4af3 100644 --- a/integration-tests/src/test/scala/org/virtuslab/yaml/OfficialTestSuiteSpec.scala +++ b/integration-tests/src/test/scala/org/virtuslab/yaml/OfficialTestSuiteSpec.scala @@ -1,7 +1,17 @@ package org.virtuslab.yaml +import scala.util.matching.Regex + class OfficialTestSuiteSpec extends YamlRunnerSpec { - override def resourcePath: String = "/yaml/test-suite" - override def createTestRunner(yamlPath: os.Path): TestRunner = YamlSuiteTestRunner(yamlPath) + override val verbose: Boolean = true + override val testOutYaml: Boolean = false + override val skipErrors: Boolean = false + override val skipInvalidEvents: Boolean = false + + // override def runMatching: Regex = ".*LQZ7.*".r + + override def resourcePath: String = "/yaml/test-suite" + override def createTestRunner(yamlPath: os.Path): TestRunner = + YamlSuiteTestRunner(yamlPath, testOutYaml) } diff --git a/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala b/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala index 407903849..60fe3a8bd 100644 --- a/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala +++ b/integration-tests/src/test/scala/org/virtuslab/yaml/YamlRunnerSpec.scala @@ -1,13 +1,19 @@ package org.virtuslab.yaml import java.io.File +import scala.util.matching.Regex abstract class YamlRunnerSpec extends munit.FunSuite { def resourcePath: String def createTestRunner(yamlPath: os.Path): TestRunner - val verbose = false + def runMatching: Regex = ".*".r + + val testOutYaml: Boolean = false + val skipErrors: Boolean = false + val skipInvalidEvents: Boolean = false + val verbose: Boolean = false val predicate: os.Path => Boolean = _ => true val yamlDirPath = getClass.getResource(resourcePath) @@ -19,53 +25,118 @@ abstract class YamlRunnerSpec extends munit.FunSuite { val yamlDir = new File(yamlDirPath.getPath) val yamlPaths: List[os.Path] = yamlDir.listFiles().map(os.Path(_)).filter(predicate).toList + case class Results( + invalidEventsPaths: List[os.Path], + errorsPaths: List[os.Path], + successes: Int + ) { + def allFailures: List[os.Path] = invalidEventsPaths ++ errorsPaths + + def count = invalidEventsPaths.size + errorsPaths.size + successes + } + test("should parse yaml to event") { - def loop(paths: List[os.Path], failsPath: List[os.Path]): List[os.Path] = { + def loop( + paths: List[os.Path], + invalidEventsPaths: List[os.Path], + errorsPaths: List[os.Path], + successes: Int + ): Results = { paths match { case path :: tail => { - val testRunner = createTestRunner(path) - if (verbose) { - println(s"Running $path") - } - val runnerResult = testRunner.run() - runnerResult match { - case RunnerResult.Success(_) => loop(tail, failsPath) - case RunnerResult.InvalidEvents(obtained, expected) => - println(s"Events differ - $path") - if (verbose) { - println(s"Obtained:\n$obtained") - println(s"Expected:\n$expected") - } - loop(tail, path :: failsPath) - case RunnerResult.Error(eventsUntilError, error) => - println(s"Error in test - $path") - if (verbose) { - println(s"Obtained:\n$eventsUntilError") - println(s"Encountered error:\n$error") - } - loop(tail, path :: failsPath) + if (!runMatching.matches(path.toString)) { + loop(tail, invalidEventsPaths, errorsPaths, successes) + } else { + val testRunner = createTestRunner(path) + if (verbose) { + println(s"Running $path") + } + val runnerResult = testRunner.run() + runnerResult match { + case RunnerResult.InvalidOutYaml(eventsResult, outYaml, expectedOutYaml) => + val escapedOutYaml = outYaml.replace("\\", "\\\\").replace("\n", "\\n") + val escapedExpectedOutYaml = + expectedOutYaml.replace("\\", "\\\\").replace("\n", "\\n") + println(s"Out yaml differ - $path") + if (verbose && !skipInvalidEvents) { + println(s"Obtained:\n$outYaml") + println(s"Expected:\n$expectedOutYaml") + println("Escaped:") + println(s"Obtained:\n$escapedOutYaml") + println(s"Expected:\n$escapedExpectedOutYaml") + } + eventsResult match { + case RunnerResult.InvalidEvents(obtained, expected) => + println(s"Events differ - $path") + if (verbose && !skipInvalidEvents) { + println(s"Obtained:\n$obtained") + println(s"Expected:\n$expected") + } + case _ => + } + loop(tail, path :: invalidEventsPaths, errorsPaths, successes) + case RunnerResult.Success(_) => + loop(tail, invalidEventsPaths, errorsPaths, successes + 1) + case RunnerResult.InvalidEvents(obtained, expected) => + println(s"Events differ - $path") + if (verbose && !skipInvalidEvents) { + println(s"Obtained:\n$obtained") + println(s"Expected:\n$expected") + } + loop(tail, path :: invalidEventsPaths, errorsPaths, successes) + case RunnerResult.Error(eventsUntilError, error) => + println(s"Error in test - $path") + if (verbose && !skipErrors) { + println(s"Obtained:\n$eventsUntilError") + println(s"Encountered error:\n$error") + } + loop(tail, invalidEventsPaths, path :: errorsPaths, successes) + } } } - case Nil => failsPath + case Nil => Results(invalidEventsPaths, errorsPaths, successes) } } - val failsPath = loop(yamlPaths, Nil) + val results = loop(yamlPaths, Nil, Nil, 0) + + // I need to refactor this summary so that it takes errors and invalid events into account + // if one of them are to be skipped, stats should disregard them (total should be smaller, failed should be of non-skipped type) + val total = results.count - val all = yamlPaths.size - val failed = failsPath.size - val passed = (yamlPaths.size - failsPath.size) + val all = + if (skipErrors && skipInvalidEvents) results.count - results.allFailures.size + else if (skipErrors) results.count - results.errorsPaths.size + else if (skipInvalidEvents) results.count - results.invalidEventsPaths.size + else results.count + + val invalidEvents = results.invalidEventsPaths.size + val errors = results.errorsPaths.size + + val skipped = if (skipErrors) invalidEvents else if (skipInvalidEvents) errors else 0 + + val failed = + if (skipErrors && skipInvalidEvents) 0 + else if (skipErrors) invalidEvents + else if (skipInvalidEvents) errors + else invalidEvents + errors + + val passed = all - failed val summary = s"""| |SUMMARY | + |Total tests: $total + |Skipped: $skipped (skipped errors: $skipErrors (encountered $errors), skipped invalid events: $skipInvalidEvents (encountered $invalidEvents)) + |Summarised tests: $all + | |Passed: $passed/$all ${"%.2f".format(100 * passed.toFloat / all.toFloat)}% |Failed: $failed/$all ${"%.2f".format(100 * failed.toFloat / all.toFloat)}% |""".stripMargin println(summary) - assert(failsPath.isEmpty) + assert(results.allFailures.isEmpty) } }