Add --dry_run to viash [ns] test for generating test scripts - On develop_0_8

CHANGELOG.md

@@ -4,6 +4,10 @@
 
 * `docker setup strategy`: Fix inconsistencies in the documentation (PR #657).
 
+## NEW FUNCTIONALITY
+
+* `viash test` and `viash ns test`: Add a hidden `--dry_run` option to build the tests without executing them (PR #676).
+
 ## BUG FIXES
 
 * `NextflowPlatform`: Fix publishing state for output arguments with `multiple: true` (#638, PR #639). 

src/main/scala/io/viash/Main.scala

@@ -244,13 +244,17 @@
       case List(cli.test) =>
         val (config, platform) = readConfig(cli.test, project = proj1)
         val config2 = singleConfigDependencies(config, platform, None, proj1.rootDir)
+        if (cli.test.dryRun.getOrElse(false) && !cli.test.keep.toOption.map(_.toBoolean).getOrElse(false)) {
+          info("Warning: --dry_run is set, but --keep is not set. This will result in the generated files being deleted after the test.")
+        }
         ViashTest(
           config2,
           platform = platform.get,
           keepFiles = cli.test.keep.toOption.map(_.toBoolean),
           setupStrategy = cli.test.setup.toOption,
           cpus = cli.test.cpus.toOption,
-          memory = cli.test.memory.toOption
+          memory = cli.test.memory.toOption,
+          dryRun = cli.test.dryRun.toOption
         )
         0 // Exceptions are thrown when a test fails, so then the '0' is not returned but a '1'. Can be improved further.
       case List(cli.namespace, cli.namespace.build) =>
@@ -271,6 +275,9 @@
       case List(cli.namespace, cli.namespace.test) =>
         val (configs, allConfigs) = readConfigs(cli.namespace.test, project = proj1)
         val configs2 = namespaceDependencies(configs, allConfigs, None, proj1.rootDir)
+        if (cli.namespace.test.dryRun.getOrElse(false) && !cli.namespace.test.keep.toOption.map(_.toBoolean).getOrElse(false)) {
+          info("Warning: --dry_run is set, but --keep is not set. This will result in the generated files being deleted after the test.")
+        }
         val testResults = ViashNamespace.test(
           configs = configs2,
           parallel = cli.namespace.test.parallel(),
@@ -280,6 +287,7 @@
           cpus = cli.namespace.test.cpus.toOption,
           memory = cli.namespace.test.memory.toOption,
           setup = cli.namespace.test.setup.toOption,
+          dryRun = cli.namespace.test.dryRun.toOption
         )
         val errors = testResults.flatMap(_.toOption).count(_.isError)
         if (errors > 0) 1 else 0

src/main/scala/io/viash/ViashNamespace.scala

@@ -113,7 +113,8 @@ object ViashNamespace extends Logging {
     tsv: Option[String] = None,
     append: Boolean = false,
     cpus: Option[Int],
-    memory: Option[String]
+    memory: Option[String],
+    dryRun: Option[Boolean] = None
   ): List[Either[(Config, ManyTestOutput), Status]] = {
     val configs1 = configs.filter{tup => tup match {
       // remove nextflow because unit testing nextflow modules
@@ -189,7 +190,8 @@ object ViashNamespace extends Logging {
                 quiet = true,
                 parentTempPath = Some(parentTempPath),
                 cpus = cpus,
-                memory = memory
+                memory = memory,
+                dryRun = dryRun
               )
             } catch {
               case e: MissingResourceFileException => 

src/main/scala/io/viash/ViashTest.scala

@@ -67,7 +67,8 @@ object ViashTest extends Logging {
     verbosityLevel: Int = 6,
     parentTempPath: Option[Path] = None, 
     cpus: Option[Int], 
-    memory: Option[String]
+    memory: Option[String],
+    dryRun: Option[Boolean] = None
   ): ManyTestOutput = {
     // create temporary directory
     val dir = IO.makeTemp("viash_test_" + config.functionality.name, parentTempPath)
@@ -98,7 +99,8 @@ object ViashTest extends Logging {
       setupStrategy = setupStrategy.getOrElse("cachedbuild"),
       verbosityLevel = verbosityLevel,
       cpus = cpus,
-      memory = memory
+      memory = memory,
+      dryRun = dryRun
     )
     val count = results.count(_.exitValue == 0)
     val anyErrors = setupRes.exists(_.exitValue > 0) || count < results.length
@@ -147,7 +149,8 @@ object ViashTest extends Logging {
     setupStrategy: String, 
     verbosityLevel: Int, 
     cpus: Option[Int], 
-    memory: Option[String]
+    memory: Option[String],
+    dryRun: Option[Boolean]
   ): ManyTestOutput = {
     val fun = config.functionality
 
@@ -317,7 +320,10 @@ object ViashTest extends Logging {
           val subTmp = Paths.get(newDir.toString, "tmp")
           Files.createDirectories(subTmp)
 
-          val cmd = Seq(executable) ++ Seq(cpus.map("---cpus=" + _), memory.map("---memory="+_)).flatMap(a => a)
+          val cmd = if (dryRun.getOrElse(false))
+            Seq("echo", "Running dummy test script")
+          else
+            Seq(executable) ++ Seq(cpus.map("---cpus=" + _), memory.map("---memory="+_)).flatMap(a => a)
           logger(s"+${cmd.mkString(" ")}")
           val exitValue = Process(
             cmd,
@@ -340,4 +346,5 @@ object ViashTest extends Logging {
 
     ManyTestOutput(buildResult, testResults)
   }
+
 }

src/main/scala/io/viash/cli/CLIConf.scala

@@ -238,14 +238,20 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin
       "viash test",
       "Test the component using the tests defined in the viash config file.",
       "viash test config.vsh.yaml [-p docker] [-k true/false] [--setup cachedbuild]")
-    
+
     val setup = registerOpt[String](
       name = "setup",
       short = Some('s'),
       default = None,
       descr = "Which @[setup strategy](docker_setup_strategy) for creating the container to use [Docker Platform only]."
     )
-
+    val dryRun = registerOpt[Boolean](
+      name= "dry_run",
+      default = Some(false),
+      descr = "Only generate the test script, do not run the test.",
+      hidden = true
+    )
+
     footer(
       s"""
          |The temporary directory can be altered by setting the VIASH_TEMP directory. Example:
@@ -336,6 +342,12 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin
         default = Some(false),
         descr = "Append to tsv instead of overwrite"
       )
+      val dryRun = registerOpt[Boolean](
+        name= "dry_run",
+        default = Some(false),
+        descr = "Only generate the test scripts, do not run them.",
+        hidden = true
+      )
     }
 
     val list = new DocumentedSubcommand("list") with ViashNs with ViashLogger {

src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala

@@ -96,6 +96,39 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
     val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r
     assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
   }
+
+  test("Check namespace test output with working dir message using --dry_run") {
+    val (stdout, stderr, _) = TestHelper.testMainWithStdErr(
+      "ns", "test",
+      "--src", nsPath,
+      "--keep", "true",
+      "--dry_run"
+    )
+
+    // Test inclusion of a header
+    val regexHeader = raw"^\s*namespace\s*functionality\s*platform\s*test_name\s*exit_code\s*duration\s*result".r
+    assert(regexHeader.findFirstIn(stdout).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n${stdout}")
+
+    val regexWdir = raw"The working directory for the namespace tests is [\w/]+[\r\n]{1,2}".r
+    assert(regexWdir.findFirstIn(stderr).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n${stderr}")
+
+    for (
+      (component, steps) <- components;
+      (step, resultPattern) <- steps
+    ) {
+      // In this mode, a test script can never return an ERROR
+      val updtResultPattern = if (resultPattern == raw"\s*1\s*\d+\s*ERROR") {
+        raw"\s*0\s*\d+\s*SUCCESS"
+      } else {
+        resultPattern
+      }
+      val regex = s"""testns\\s*$component\\s*native\\s*$step$updtResultPattern""".r
+      assert(regex.findFirstIn(stdout).isDefined, s"\nRegex: '${regex.toString}'; text: \n${stdout}")
+    }
+
+    val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r
+    assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
+  }
 
   test("Check namespace test output with tsv option") {
     val log = Paths.get(tempFolStr, "log.tsv").toFile

src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala

@@ -228,6 +228,21 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
     checkTempDirAndRemove(testText, true)
   }
 
+  test("Check output in case --dry_run is specified") {
+    val testText = TestHelper.testMain(
+      "test",
+      "-p", "native",
+      "--dry_run",
+      configFile
+    )
+    assert(testText.contains("Running tests in temporary directory: "))
+    assert(testText.contains("Running dummy test script"))
+    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testText.contains("Cleaning up temporary directory"))
+
+    checkTempDirAndRemove(testText, false)
+  }
+
   test("Check output in case --keep false is specified") {
     val testText = TestHelper.testMain(
       "test",