diff --git a/.github/workflows/sbt_test.yml b/.github/workflows/sbt_test.yml index 54a9a3547..6182f0ead 100644 --- a/.github/workflows/sbt_test.yml +++ b/.github/workflows/sbt_test.yml @@ -42,12 +42,19 @@ jobs: processx testthat - - name: Set up sbt + - name: Set up java & sbt uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java.ver }} + - name: Set up sbt specifically on macOS on arm64 if needed + if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64'}} + run: | + if ! command -v sbt &> /dev/null; then + brew install sbt + fi + - name: Set up Scala run: | if [[ "${{ matrix.config.os }}" =~ ^macos.*$ ]]; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b8e10f29..ea727c960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,44 @@ +# Viash 0.8.6 (2024-04-26): Bug fixes and improvements for CI + +Fix some issues in some edge cases. +Add options for testing in a CI environment. Given that these options are not meant for general use, they are hidden from the help message. +Some improvements are made to run in Nextflow Fusion. + +## DOCUMENTATION + +* `docker setup strategy`: Fix inconsistencies in the documentation (PR #657). + +* `repositories`: Fix `uri` -> `repo` in the repositories documentation (PR #682). + +## NEW FUNCTIONALITY + +* `viash test` and `viash ns test`: Add a hidden `--dry_run` option to build the tests without executing them (PR #676). + +* `viash test` and `viash ns test`: Add a hidden `--deterministic_working directory` argument to use a fixed directory path (PR #683). + +* `component names`: Verify that component namespace and name combinations are unique (PR #685). + +## BUG FIXES + +* `NextflowPlatform`: Fix publishing state for output arguments with `multiple: true` (#638, PR #639). + +* `Executable`: Check whether a multiple output file argument contains a wildcard (PR #639). + +* `NextflowPlatform`: Fix a possible cause of concurrency issues (PR #669). + +* `Resources`: Fix an issue where if the first resource is not a script, the resource is silently dropped (PR #670). + +* `Docker automount`: Prevent adding a trailing slash to an automounted folder (PR #673). + +* `NextflowPlatform`: Change the at-runtime generated nextflow process from an in-memory to an on-disk temporary file, which should cause less issues with Nextflow Fusion (PR #681). + # Viash 0.8.5 (2024-02-21): Bug fixes and documentation improvements Fix a bug when building a test docker container which requires a test resource. Additional improvements for the website documentation and support for the latest version of Nextflow are added. ## BUG FIXES -* `nextflow runner`: Fix an issue with current nextflow-latest (24.01.0-edge) where our supporting library passes a GString instead of a String and results in a type mismatch (PR #640). +* `NextflowPlatform`: Fix an issue with current nextflow-latest (24.01.0-edge) where our supporting library passes a GString instead of a String and results in a type mismatch (PR #640). * `test resources`: Make non-script test resources available during building of a docker container for `viash test` (PR #652). diff --git a/build.sbt b/build.sbt index fb5ab9abc..14a4e3b8e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "viash" -version := "0.8.5" +version := "0.8.6" scalaVersion := "2.13.10" diff --git a/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh b/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh index d47d74983..e46a4ed51 100644 --- a/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh +++ b/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh @@ -15,7 +15,11 @@ function ViashAutodetectMount { base_name=`basename "$abs_path"` fi mount_target="/viash_automount$mount_source" - echo "$mount_target/$base_name" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi } function ViashAutodetectMountArg { abs_path=$(ViashAbsolutePath "$1") diff --git a/src/main/resources/io/viash/platforms/nextflow/DataflowHelper.nf b/src/main/resources/io/viash/platforms/nextflow/DataflowHelper.nf index 464a3062d..450ad7492 100644 --- a/src/main/resources/io/viash/platforms/nextflow/DataflowHelper.nf +++ b/src/main/resources/io/viash/platforms/nextflow/DataflowHelper.nf @@ -52,16 +52,16 @@ def setWorkflowArguments(Map args) { // determine splitargs def splitArgs = args. collectEntries{procKey, dataKeys -> - // dataKeys is a map but could also be a list - newSplitData = dataKeys - .collectEntries{ val -> - newKey = val instanceof String ? val : val.key - origKey = val instanceof String ? val : val.value - [ newKey, data[origKey] ] - } - .findAll{it.value} - [procKey, newSplitData] - } + // dataKeys is a map but could also be a list + def newSplitData = dataKeys + .collectEntries{ val -> + newKey = val instanceof String ? val : val.key + origKey = val instanceof String ? val : val.value + [ newKey, data[origKey] ] + } + .findAll{it.value} + [procKey, newSplitData] + } // return output [ id, newData, splitArgs] + passthrough diff --git a/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf b/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf index b774bfffb..306a74e93 100644 --- a/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf +++ b/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf @@ -84,7 +84,7 @@ def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { .findAll { it.type == "file" && it.direction == "output" } .indexed() .collectEntries{ index, par -> - out = output[index + 1] + def out = output[index + 1] // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) if (!out instanceof List || out.size() <= 1) { if (par.multiple) { @@ -210,7 +210,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { def inputFileExports = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } .collect { par -> - viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})" + def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})" "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}" } @@ -266,7 +266,6 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { | .join("\\n") |\"\"\" |# meta exports - |# export VIASH_META_RESOURCES_DIR="\${resourcesDir.toRealPath().toAbsolutePath()}" |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" |export VIASH_META_FUNCTIONALITY_NAME="${meta.config.functionality.name}" @@ -308,19 +307,22 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { // println("######################\n$procStr\n######################") // } - // create runtime process - def ownerParams = new nextflow.script.ScriptBinding.ParamsMap() - def binding = new nextflow.script.ScriptBinding().setParams(ownerParams) - def module = new nextflow.script.IncludeDef.Module(name: procKey) + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) def session = nextflow.Nextflow.getSession() - def scriptParser = new nextflow.script.ScriptParser(session) + def parser = new nextflow.script.ScriptParser(session) .setModule(true) .setBinding(binding) - scriptParser.scriptPath = scriptMeta.getScriptPath() - def moduleScript = scriptParser.runScript(procStr) + def moduleScript = parser.runScript(tempFile) .getScript() // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) scriptMeta.addModule(moduleScript, module.name, module.alias) // retrieve and return process from meta diff --git a/src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf b/src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf index ca8125d75..569363331 100644 --- a/src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf +++ b/src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf @@ -114,7 +114,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi } expectedClass = value instanceof Double ? null : "Double" } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { - // cast to boolean if need ben + // cast to boolean if need be if (value instanceof String) { def valueLower = value.toLowerCase() if (valueLower == "true") { diff --git a/src/main/resources/io/viash/platforms/nextflow/channel/_splitParams.nf b/src/main/resources/io/viash/platforms/nextflow/channel/_splitParams.nf index 440380474..6bf055e87 100644 --- a/src/main/resources/io/viash/platforms/nextflow/channel/_splitParams.nf +++ b/src/main/resources/io/viash/platforms/nextflow/channel/_splitParams.nf @@ -18,13 +18,13 @@ Map _splitParams(Map parValues, Map config){ } if (parameterSettings.multiple) { // Check if parameter can accept multiple values if (parValue instanceof Collection) { - parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } } else if (parValue instanceof String) { - parValue = parValue.split(parameterSettings.multiple_sep) + parValue = parValue.split(parameterSettings.multiple_sep) } else if (parValue == null) { - parValue = [] + parValue = [] } else { - parValue = [ parValue ] + parValue = [ parValue ] } parValue = parValue.flatten() } diff --git a/src/main/resources/io/viash/platforms/nextflow/channel/channelFromParams.nf b/src/main/resources/io/viash/platforms/nextflow/channel/channelFromParams.nf index 0d7881668..21d486382 100644 --- a/src/main/resources/io/viash/platforms/nextflow/channel/channelFromParams.nf +++ b/src/main/resources/io/viash/platforms/nextflow/channel/channelFromParams.nf @@ -60,9 +60,9 @@ private List>> _paramsToParamSets(Map params, // Process ID def id = tup[0] ?: globalID - if (workflow.stubRun) { + if (workflow.stubRun && !id) { // if stub run, explicitly add an id if missing - id = id ? id : "stub${index}" + id = "stub${index}" } assert id != null: "Each parameter set should have at least an 'id'" diff --git a/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf b/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf index 0effeb6fb..e8467187c 100644 --- a/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf +++ b/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf @@ -77,6 +77,7 @@ def runComponents(Map args) { ? out_ch | map{tup -> def output = tup[1] def old_state = tup[2] + def new_state = null if (toState_ instanceof Map) { new_state = old_state + toState_.collectEntries{ key0, key1 -> [key0, output[key1]] diff --git a/src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf b/src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf index d3a339561..a6a1223f0 100644 --- a/src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf +++ b/src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf @@ -9,7 +9,7 @@ * @return Whether the path is absolute, as a boolean. */ def _stringIsAbsolutePath(path) { - _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ assert path instanceof String return _resolve_URL_PROTOCOL.matcher(path).matches() diff --git a/src/main/resources/io/viash/platforms/nextflow/readwrite/readCsv.nf b/src/main/resources/io/viash/platforms/nextflow/readwrite/readCsv.nf index 9726f35e7..0b33f7013 100644 --- a/src/main/resources/io/viash/platforms/nextflow/readwrite/readCsv.nf +++ b/src/main/resources/io/viash/platforms/nextflow/readwrite/readCsv.nf @@ -38,7 +38,7 @@ def readCsv(file_path) { if (field == "") { return null } - m = removeQuote.matcher(field) + def m = removeQuote.matcher(field) if (m.find()) { return m.replaceFirst('$1') } else { diff --git a/src/main/resources/io/viash/platforms/nextflow/states/findStates.nf b/src/main/resources/io/viash/platforms/nextflow/states/findStates.nf index 1e1b2b17c..226691bca 100644 --- a/src/main/resources/io/viash/platforms/nextflow/states/findStates.nf +++ b/src/main/resources/io/viash/platforms/nextflow/states/findStates.nf @@ -1,5 +1,4 @@ def findStates(Map params, Map config) { - // TODO: do a deep clone of config def auto_config = deepClone(config) def auto_params = deepClone(params) diff --git a/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf b/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf index 4e556b84d..cbc36b509 100644 --- a/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf +++ b/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf @@ -53,20 +53,20 @@ def publishStates(Map args) { def state_ = tup[1] // the input files and the target output filenames - def inputOutputFiles_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputOutputFiles_[0] - def outputFiles_ = inputOutputFiles_[1] + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] def yamlFilename = yamlTemplate_ .replaceAll('\\$id', id_) .replaceAll('\\$key', key_) - // TODO: do the pathnames in state_ match up with the outputFiles_? + // TODO: do the pathnames in state_ match up with the outputFilenames_? // convert state to yaml blob def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFiles_] + [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_] } | publishStatesProc emit: input_ch @@ -99,12 +99,12 @@ process publishStatesProc { } } """ - mkdir -p "\$(dirname '${yamlFile}')" - echo "Storing state as yaml" - echo '${yamlBlob}' > '${yamlFile}' - echo "Copying output files to destination folder" - ${copyCommands.join("\n ")} - """ +mkdir -p "\$(dirname '${yamlFile}')" +echo "Storing state as yaml" +echo '${yamlBlob}' > '${yamlFile}' +echo "Copying output files to destination folder" +${copyCommands.join("\n ")} +""" } @@ -133,9 +133,13 @@ def publishStatesByConfig(Map args) { .replaceAll('\\$key', key_) def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - // the processed state is a list of [key, value, srcPath, destPath] tuples, where - // - key, value is part of the state to be saved to disk - // - srcPath and destPath are lists of files to be copied from src to dest + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (key, value) are the tuples that will be saved to the state.yaml file + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) def processedState = config.functionality.allArguments .findAll { it.direction == "output" } @@ -152,7 +156,7 @@ def publishStatesByConfig(Map args) { // in the state as-is, but is not something that needs // to be copied from the source path to the dest path if (par.type != "file") { - return [[key: plainName_, value: value, srcPath: [], destPath: []]] + return [[key: plainName_, value: value, inputPath: [], outputFilename: []]] } // if the orig state does not contain this filename, // it's an optional argument for which the user specified @@ -175,15 +179,16 @@ def publishStatesByConfig(Map args) { // the index of the file assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" def outputPerFile = value.withIndex().collect{ val, ix -> - def value_ = java.nio.file.Paths.get(filename.replace("*", ix.toString())) + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) // if id contains a slash if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def srcPath = val instanceof File ? val.toPath() : val - [value: value_, srcPath: srcPath, destPath: destPath] + def inputPath = val instanceof File ? val.toPath() : val + [value: value_, inputPath: inputPath, outputFilename: filename_ix] } - def transposedOutputs = ["value", "srcPath", "destPath"].collectEntries{ key -> + def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> [key, outputPerFile.collect{dic -> dic[key]}] } return [[key: plainName_] + transposedOutputs] @@ -193,22 +198,22 @@ def publishStatesByConfig(Map args) { if (yamlDir != null) { value_ = yamlDir.relativize(value_) } - def srcPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: value_, srcPath: [srcPath], destPath: [filename]]] + def inputPath = value instanceof File ? value.toPath() : value + return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]] } } def updatedState_ = processedState.collectEntries{[it.key, it.value]} - def inputFiles_ = processedState.collectMany{it.srcPath} - def outputFiles_ = processedState.collectMany{it.destPath} + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - [id_, yamlBlob_, yamlFilename, inputFiles_, outputFiles_] + [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames] } | publishStatesProc emit: input_ch } return publishStatesSimpleWf -} \ No newline at end of file +} diff --git a/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf b/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf index 7061c42d8..26c50ad69 100644 --- a/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf +++ b/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf @@ -15,7 +15,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { take: input_ main: - chModified = input_ + def chModified = input_ | checkUniqueIds([:]) | _debug(workflowArgs, "input") | map { tuple -> @@ -102,12 +102,14 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { tuple } - chModifiedFiltered = workflowArgs.filter ? + def chModifiedFiltered = workflowArgs.filter ? chModified | filter{workflowArgs.filter(it)} : chModified + def chRun = null + def chPassthrough = null if (workflowArgs.runIf) { - runIfBranch = chModifiedFiltered.branch{ tup -> + def runIfBranch = chModifiedFiltered.branch{ tup -> run: workflowArgs.runIf(tup[0], tup[1]) passthrough: true } @@ -118,7 +120,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { chPassthrough = Channel.empty() } - chArgs = workflowArgs.fromState ? + def chArgs = workflowArgs.fromState ? chRun | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] @@ -126,7 +128,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { chRun | map {tup -> tup.take(2)} // fill in defaults - chArgsWithDefaults = chArgs + def chArgsWithDefaults = chArgs | map { tuple -> def id_ = tuple[0] def data_ = tuple[1] @@ -164,7 +166,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - chInitialOutput = chArgsWithDefaults + def chInitialOutput = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) @@ -194,7 +196,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { // | view{"chInitialOutput: ${it.take(3)}"} // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) + def chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) // input tuple format: [join_id, id, output, prev_state, ...] // output tuple format: [join_id, id, new_state, ...] | map{ tup -> @@ -203,7 +205,7 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) { } if (workflowArgs.auto.publish == "state") { - chPublish = chNewState + def chPublish = chNewState // input tuple format: [join_id, id, new_state, ...] // output tuple format: [join_id, id, new_state] | map{ tup -> diff --git a/src/main/scala/io/viash/Main.scala b/src/main/scala/io/viash/Main.scala index 858592407..01ccb2420 100644 --- a/src/main/scala/io/viash/Main.scala +++ b/src/main/scala/io/viash/Main.scala @@ -244,13 +244,18 @@ object Main extends Logging { case List(cli.test) => val (config, platform) = readConfig(cli.test, project = proj1) val config2 = singleConfigDependencies(config, platform, None, proj1.rootDir) + if (cli.test.dryRun.getOrElse(false) && !cli.test.keep.toOption.map(_.toBoolean).getOrElse(false)) { + info("Warning: --dry_run is set, but --keep is not set. This will result in the generated files being deleted after the test.") + } ViashTest( config2, platform = platform.get, keepFiles = cli.test.keep.toOption.map(_.toBoolean), setupStrategy = cli.test.setup.toOption, cpus = cli.test.cpus.toOption, - memory = cli.test.memory.toOption + memory = cli.test.memory.toOption, + dryRun = cli.test.dryRun.toOption, + deterministicWorkingDirectory = cli.test.deterministicWorkingDirectory.toOption ) 0 // Exceptions are thrown when a test fails, so then the '0' is not returned but a '1'. Can be improved further. case List(cli.namespace, cli.namespace.build) => @@ -271,6 +276,9 @@ object Main extends Logging { case List(cli.namespace, cli.namespace.test) => val (configs, allConfigs) = readConfigs(cli.namespace.test, project = proj1) val configs2 = namespaceDependencies(configs, allConfigs, None, proj1.rootDir) + if (cli.namespace.test.dryRun.getOrElse(false) && !cli.namespace.test.keep.toOption.map(_.toBoolean).getOrElse(false)) { + info("Warning: --dry_run is set, but --keep is not set. This will result in the generated files being deleted after the test.") + } val testResults = ViashNamespace.test( configs = configs2, parallel = cli.namespace.test.parallel(), @@ -280,6 +288,8 @@ object Main extends Logging { cpus = cli.namespace.test.cpus.toOption, memory = cli.namespace.test.memory.toOption, setup = cli.namespace.test.setup.toOption, + dryRun = cli.namespace.test.dryRun.toOption, + deterministicWorkingDirectory = cli.namespace.test.deterministicWorkingDirectory.toOption, ) val errors = testResults.flatMap(_.toOption).count(_.isError) if (errors > 0) 1 else 0 @@ -390,8 +400,7 @@ object Main extends Logging { Some(ViashNamespace.targetOutputPath( targetDir = td, platformId = pl, - namespace = config.functionality.namespace, - functionalityName = config.functionality.name + config = config )) case _ => None } diff --git a/src/main/scala/io/viash/ViashNamespace.scala b/src/main/scala/io/viash/ViashNamespace.scala index 451bf02e1..da930ccff 100644 --- a/src/main/scala/io/viash/ViashNamespace.scala +++ b/src/main/scala/io/viash/ViashNamespace.scala @@ -51,6 +51,9 @@ object ViashNamespace extends Logging { list.foreach(f) } + def targetOutputPath(targetDir: String, platformId: String, config: Config): String = + targetOutputPath(targetDir, platformId, config.functionality.namespace, config.functionality.name) + def targetOutputPath( targetDir: String, platformId: String, @@ -86,7 +89,7 @@ object ViashNamespace extends Logging { if (flatten) { target } else { - targetOutputPath(target, platformId, ns, funName) + targetOutputPath(target, platformId, conf) } val nsStr = ns.map(" (" + _ + ")").getOrElse("") infoOut(s"Exporting $funName$nsStr =$platformId=> $out") @@ -113,7 +116,9 @@ object ViashNamespace extends Logging { tsv: Option[String] = None, append: Boolean = false, cpus: Option[Int], - memory: Option[String] + memory: Option[String], + dryRun: Option[Boolean] = None, + deterministicWorkingDirectory: Option[String] = None ): List[Either[(Config, ManyTestOutput), Status]] = { val configs1 = configs.filter{tup => tup match { // remove nextflow because unit testing nextflow modules @@ -134,7 +139,11 @@ object ViashNamespace extends Logging { val tsvExists = tsvPath.exists(Files.exists(_)) val tsvWriter = tsvPath.map(Files.newBufferedWriter(_, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND)) - val parentTempPath = IO.makeTemp("viash_ns_test") + val parentTempPath = IO.makeTemp( + name = deterministicWorkingDirectory.getOrElse("viash_ns_test"), + parentTempPath = None, + addRandomized = deterministicWorkingDirectory.isEmpty + ) if (keepFiles.getOrElse(true)) { info(s"The working directory for the namespace tests is ${parentTempPath.toString()}") } @@ -173,6 +182,7 @@ object ViashNamespace extends Logging { val namespace = conf.functionality.namespace.getOrElse("") val funName = conf.functionality.name val platName = platform.id + val directoryName = if(namespace.isEmpty) funName else namespace.replace('/', '_') + "_" + funName // print start message infoOut("%20s %20s %20s %20s %9s %8s %20s".format(namespace, funName, platName, "start", "", "", "")) @@ -189,7 +199,9 @@ object ViashNamespace extends Logging { quiet = true, parentTempPath = Some(parentTempPath), cpus = cpus, - memory = memory + memory = memory, + dryRun = dryRun, + deterministicWorkingDirectory = Some(directoryName) ) } catch { case e: MissingResourceFileException => diff --git a/src/main/scala/io/viash/ViashTest.scala b/src/main/scala/io/viash/ViashTest.scala index 56f7df88d..afbbc601a 100644 --- a/src/main/scala/io/viash/ViashTest.scala +++ b/src/main/scala/io/viash/ViashTest.scala @@ -67,10 +67,16 @@ object ViashTest extends Logging { verbosityLevel: Int = 6, parentTempPath: Option[Path] = None, cpus: Option[Int], - memory: Option[String] + memory: Option[String], + dryRun: Option[Boolean] = None, + deterministicWorkingDirectory: Option[String] = None, ): ManyTestOutput = { // create temporary directory - val dir = IO.makeTemp("viash_test_" + config.functionality.name, parentTempPath) + val dir = IO.makeTemp( + name = deterministicWorkingDirectory.getOrElse(s"viash_test_${config.functionality.name}_"), + parentTempPath = parentTempPath, + addRandomized = deterministicWorkingDirectory.isEmpty + ) if (!quiet) infoOut(s"Running tests in temporary directory: '$dir'") // set version to temporary value @@ -98,7 +104,8 @@ object ViashTest extends Logging { setupStrategy = setupStrategy.getOrElse("cachedbuild"), verbosityLevel = verbosityLevel, cpus = cpus, - memory = memory + memory = memory, + dryRun = dryRun ) val count = results.count(_.exitValue == 0) val anyErrors = setupRes.exists(_.exitValue > 0) || count < results.length @@ -147,7 +154,8 @@ object ViashTest extends Logging { setupStrategy: String, verbosityLevel: Int, cpus: Option[Int], - memory: Option[String] + memory: Option[String], + dryRun: Option[Boolean] ): ManyTestOutput = { val fun = config.functionality @@ -317,7 +325,10 @@ object ViashTest extends Logging { val subTmp = Paths.get(newDir.toString, "tmp") Files.createDirectories(subTmp) - val cmd = Seq(executable) ++ Seq(cpus.map("---cpus=" + _), memory.map("---memory="+_)).flatMap(a => a) + val cmd = if (dryRun.getOrElse(false)) + Seq("echo", "Running dummy test script") + else + Seq(executable) ++ Seq(cpus.map("---cpus=" + _), memory.map("---memory="+_)).flatMap(a => a) logger(s"+${cmd.mkString(" ")}") val exitValue = Process( cmd, @@ -340,4 +351,5 @@ object ViashTest extends Logging { ManyTestOutput(buildResult, testResults) } + } diff --git a/src/main/scala/io/viash/cli/CLIConf.scala b/src/main/scala/io/viash/cli/CLIConf.scala index 950d9b1b5..f6865750e 100644 --- a/src/main/scala/io/viash/cli/CLIConf.scala +++ b/src/main/scala/io/viash/cli/CLIConf.scala @@ -238,14 +238,26 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin "viash test", "Test the component using the tests defined in the viash config file.", "viash test config.vsh.yaml [-p docker] [-k true/false] [--setup cachedbuild]") - + val setup = registerOpt[String]( name = "setup", short = Some('s'), default = None, descr = "Which @[setup strategy](docker_setup_strategy) for creating the container to use [Docker Platform only]." ) - + val dryRun = registerOpt[Boolean]( + name= "dry_run", + default = Some(false), + descr = "Only generate the test script, do not run the test.", + hidden = true + ) + val deterministicWorkingDirectory = registerOpt[String]( + name = "deterministic_working_directory", + default = None, + descr = "Name of the working directory in the temporary directory. If not set, the working directory name will be a partially randomized name.", + hidden = true + ) + footer( s""" |The temporary directory can be altered by setting the VIASH_TEMP directory. Example: @@ -336,6 +348,18 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin default = Some(false), descr = "Append to tsv instead of overwrite" ) + val dryRun = registerOpt[Boolean]( + name= "dry_run", + default = Some(false), + descr = "Only generate the test scripts, do not run them.", + hidden = true + ) + val deterministicWorkingDirectory = registerOpt[String]( + name = "deterministic_working_directory", + default = None, + descr = "Name of the working directory in the temporary directory. If not set, the working directory name will be a partially randomized name.", + hidden = true + ) } val list = new DocumentedSubcommand("list") with ViashNs with ViashLogger { diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index 3a474c73b..39dc8640e 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -24,6 +24,7 @@ import io.viash.helpers.{Git, GitInfo, IO, Logging} import io.viash.helpers.circe._ import io.viash.helpers.status._ import io.viash.helpers.Yaml +import io.viash.ViashNamespace.targetOutputPath import java.net.URI import io.viash.functionality.resources._ @@ -365,6 +366,16 @@ object Config extends Logging { val allConfigs = allConfigsWithStdOutErr.collect({ case Left((config, _, _)) if config.functionality.isEnabled => config }) + // Verify that all configs except the disabled ones are unique + // Even configs disabled by the query should be unique as they might be picked up as a dependency + // Only allowed exception are configs where the status is set to disabled + val uniqueConfigs = allConfigs.groupBy(c => targetOutputPath("", "", c)) + val duplicateConfigs = uniqueConfigs.filter(_._2.size > 1) + if (duplicateConfigs.nonEmpty) { + val duplicateNames = duplicateConfigs.keys.map(_.dropWhile(_ == '/')).toSeq.sorted.mkString(", ") + throw new RuntimeException(s"Duplicate component name${ if (duplicateConfigs.size == 1) "" else "s" } found: $duplicateNames") + } + (filteredConfigs, allConfigs) } } diff --git a/src/main/scala/io/viash/functionality/Functionality.scala b/src/main/scala/io/viash/functionality/Functionality.scala index eaf657381..790af8b53 100644 --- a/src/main/scala/io/viash/functionality/Functionality.scala +++ b/src/main/scala/io/viash/functionality/Functionality.scala @@ -372,9 +372,10 @@ case class Functionality( } def mainCode: Option[String] = mainScript.flatMap(_.read) // provide function to use resources.tail but that allows resources to be an empty list + // If mainScript ends up being None because the first resource isn't a script, return the whole list def additionalResources = resources match { - case _ :: tail => tail - case _ => List.empty[Resource] + case head :: tail if head.isInstanceOf[Script] => tail + case list => list } def isEnabled: Boolean = status != Status.Disabled diff --git a/src/main/scala/io/viash/functionality/arguments/FileArgument.scala b/src/main/scala/io/viash/functionality/arguments/FileArgument.scala index 1a917896a..bdb71471b 100644 --- a/src/main/scala/io/viash/functionality/arguments/FileArgument.scala +++ b/src/main/scala/io/viash/functionality/arguments/FileArgument.scala @@ -121,7 +121,21 @@ case class FileArgument( @default("Input") direction: Direction = Input, - @description("Treat the argument value as an array. Arrays can be passed using the delimiter `--foo=1:2:3` or by providing the same argument multiple times `--foo 1 --foo 2`. You can use a custom delimiter by using the [`multiple_sep`](#multiple_sep) property. `false` by default.") + @description( + """Allow for multiple values (`false` by default). + | + |For input arguments, this will be treated as a list of values. For example, values + |can be passed using the delimiter `--foo=1:2:3` or by providing the same argument + |multiple times `--foo 1 --foo 2`. You can use a custom delimiter by using the + |[`multiple_sep`](#multiple_sep) property. + | + |For output file arguments, the passed value needs to contain a wildcard. For example, + |`--foo 'foo_*.txt'` will be treated as a list of files that match the pattern. Note that in Bash, + | the wildcard will need to be in quotes (`"foo_*.txt"` or `'foo_*.txt'`) or else Bash will + | automatically attempt to expand the expression. + | + |Other output arguments (e.g. integer, double, ...) are not supported yet. + |""".stripMargin) @example( """- name: --my_files | type: file diff --git a/src/main/scala/io/viash/functionality/dependencies/Repository.scala b/src/main/scala/io/viash/functionality/dependencies/Repository.scala index 21d7e7741..489455243 100644 --- a/src/main/scala/io/viash/functionality/dependencies/Repository.scala +++ b/src/main/scala/io/viash/functionality/dependencies/Repository.scala @@ -35,7 +35,7 @@ import java.nio.file.Files """repositories: | - name: openpipelines-bio | type: github - | uri: openpipelines-bio/modules + | repo: openpipelines-bio/modules | tag: 0.3.0 |""".stripMargin, "yaml") diff --git a/src/main/scala/io/viash/helpers/DependencyResolver.scala b/src/main/scala/io/viash/helpers/DependencyResolver.scala index 24659f18c..389b740a2 100644 --- a/src/main/scala/io/viash/helpers/DependencyResolver.scala +++ b/src/main/scala/io/viash/helpers/DependencyResolver.scala @@ -156,7 +156,7 @@ object DependencyResolver { case "nextflow" => "main.nf" case _ => c.functionality.name } - Paths.get(ViashNamespace.targetOutputPath("", pid, c.functionality.namespace, c.functionality.name), executableName).toString() + Paths.get(ViashNamespace.targetOutputPath("", pid, c), executableName).toString() } val info = c.info.get.copy( executable = executable diff --git a/src/main/scala/io/viash/helpers/IO.scala b/src/main/scala/io/viash/helpers/IO.scala index 4b2f0d7a1..42b8d3579 100644 --- a/src/main/scala/io/viash/helpers/IO.scala +++ b/src/main/scala/io/viash/helpers/IO.scala @@ -55,12 +55,26 @@ object IO extends Logging { * * @param name the name of the temporary directory * @param parentTempPath the optional parent directory for the temporary directory + * @param addRandomized enable randomization of the temporary directory name * @return the temporary directory path */ - def makeTemp(name: String, parentTempPath: Option[Path] = None): Path = { + def makeTemp(name: String, parentTempPath: Option[Path] = None, addRandomized: Boolean = true): Path = { val workTempDir = parentTempPath.getOrElse(this.tempDir) if (!Files.exists(workTempDir)) Files.createDirectories(workTempDir) - val temp = Files.createTempDirectory(workTempDir, name) + val temp = addRandomized match { + case true => Files.createTempDirectory(workTempDir, name) + case false => { + val temp = workTempDir.resolve(name) + val tempFile = temp.toFile() + if (tempFile.exists() && !tempFile.isDirectory()) { + throw new RuntimeException(s"Temporary directory $temp already exists as a file.") + } + if (tempFile.exists() && tempFile.isDirectory() && tempFile.list().length > 0 ){ + throw new RuntimeException(s"Temporary directory $temp already exists and is not empty.") + } + temp + } + } Files.createDirectories(temp) temp } diff --git a/src/main/scala/io/viash/platforms/DockerPlatform.scala b/src/main/scala/io/viash/platforms/DockerPlatform.scala index 75c3bd8cc..5a8d51893 100644 --- a/src/main/scala/io/viash/platforms/DockerPlatform.scala +++ b/src/main/scala/io/viash/platforms/DockerPlatform.scala @@ -122,11 +122,11 @@ case class DockerPlatform( +| `ifneedbebuild` | Build the image if it does not exist locally. +| `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled. +| `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). - +| `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it doesn't exist. - +| `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it doesn't exist. + +| `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it does not exist. + +| `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it does not exist. +| `ifneedbepull` | If the image does not exist locally, pull the image. - +| `ifneedbepullelsebuild` | If the image does not exist locally, pull the image. If the image does exist, build it. - +| `ifneedbepullelsecachedbuild` | If the image does not exist locally, pull the image. If the image does exist, build it with caching enabled. + +| `ifneedbepullelsebuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image from scratch. + +| `ifneedbepullelsecachedbuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image with caching enabled. +| `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). +| `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet. +| `donothing` / `meh` | Do not build or pull anything. @@ -476,7 +476,7 @@ case class DockerPlatform( val preRun = if (resolve_volume == Automatic) { val detectMounts = args.flatMap { - case arg: FileArgument if arg.multiple => + case arg: FileArgument if arg.multiple && arg.direction == Input => // resolve arguments with multiplicity different from singular args val viash_temp = "VIASH_TEST_" + arg.plainName.toUpperCase() val chownIfOutput = if (arg.direction == Output) "\n VIASH_CHOWN_VARS+=( \"$var\" )" else "" @@ -611,7 +611,7 @@ case class DockerPlatform( |function ViashPerformChown { | if (( $${#VIASH_CHOWN_VARS[@]} )); then | set +e - | eval docker run --entrypoint=chown $dockerArgs$volExtraParams $fullImageID "$$(id -u):$$(id -g)" --silent --recursive $${VIASH_CHOWN_VARS[@]} + | eval docker run --entrypoint=bash $dockerArgs$volExtraParams $fullImageID -c "'chown $$(id -u):$$(id -g) --silent --recursive $${VIASH_CHOWN_VARS[@]}'" | set -e | fi |} diff --git a/src/main/scala/io/viash/platforms/NextflowPlatform.scala b/src/main/scala/io/viash/platforms/NextflowPlatform.scala index 2f9ba1acc..a4d171904 100644 --- a/src/main/scala/io/viash/platforms/NextflowPlatform.scala +++ b/src/main/scala/io/viash/platforms/NextflowPlatform.scala @@ -245,6 +245,7 @@ case class NextflowPlatform( NextflowHelper.vdsl3Helper } + // TODO: might need to change resources_dir to just moduleDir NextflowHelper.generateHeader(config) + "\n\n" + NextflowHelper.workflowHelper + s""" @@ -255,7 +256,7 @@ case class NextflowPlatform( | |// create meta object |meta = [ - | "resources_dir": moduleDir.toRealPath().normalize(), + | "resources_dir": moduleDir, | "config": ${NextflowHelper.generateConfigStr(config)} |] | @@ -277,7 +278,6 @@ case class NextflowPlatform( |// anonymous workflow for running this module as a standalone |workflow { | // add id argument if it's not already in the config - | // TODO: deep copy | def newConfig = deepClone(meta.config) | def newParams = deepClone(params) | diff --git a/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala b/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala index d025c2a8a..f7ce66e1e 100644 --- a/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala +++ b/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala @@ -221,7 +221,7 @@ object NextflowHelper { def renderDependencies(config: Config): String = { // TODO ideally we'd already have 'thisPath' precalculated but until that day, calculate it here - val thisPath = Paths.get(ViashNamespace.targetOutputPath("", "invalid_platform_name", config.functionality.namespace, config.functionality.name)) + val thisPath = Paths.get(ViashNamespace.targetOutputPath("", "invalid_platform_name", config)) val depStrs = config.functionality.dependencies.map{ dep => NextflowHelper.renderInclude(dep, thisPath) diff --git a/src/main/scala/io/viash/wrapper/BashWrapper.scala b/src/main/scala/io/viash/wrapper/BashWrapper.scala index 1de46cbcd..7a5f24c7d 100644 --- a/src/main/scala/io/viash/wrapper/BashWrapper.scala +++ b/src/main/scala/io/viash/wrapper/BashWrapper.scala @@ -231,7 +231,7 @@ object BashWrapper { val localDependenciesStrings = localDependencies.map{ d => // relativize the path of the main component to the local dependency // TODO ideally we'd already have 'thisPath' precalculated but until that day, calculate it here - val thisPath = ViashNamespace.targetOutputPath("", "invalid_platform_name", config.functionality.namespace, config.functionality.name) + val thisPath = ViashNamespace.targetOutputPath("", "invalid_platform_name", config) val relativePath = Paths.get(thisPath).relativize(Paths.get(d.configInfo.getOrElse("executable", ""))) s"${d.VIASH_DEP}=\"$$VIASH_META_RESOURCES_DIR/$relativePath\"" } @@ -361,7 +361,12 @@ object BashWrapper { (part1 :: moreParts).mkString("\n") case param => - val multisep = if (param.multiple) Some(param.multiple_sep) else None + val multisep = + if (param.multiple && param.direction == Input) { + Some(param.multiple_sep) + } else { + None + } // params of the form --param ... val part1 = param.flags match { @@ -389,7 +394,7 @@ object BashWrapper { } else { "\n# storing leftover values in positionals\n" + positionals.map { param => - if (param.multiple) { + if (param.multiple && param.direction == Input) { s"""while [[ $$# -gt 0 ]]; do | ${store("positionalArg", param.VIASH_PAR, "\"$1\"", Some(param.multiple_sep)).mkString("\n ")} | shift 1 @@ -456,7 +461,12 @@ object BashWrapper { } else { "\n# check whether required files exist\n" + files.map { param => - if (param.multiple) { + if (!param.multiple) { + s"""if [ ! -z "$$${param.VIASH_PAR}" ] && [ ! -e "$$${param.VIASH_PAR}" ]; then + | ViashError "$direction file '$$${param.VIASH_PAR}' does not exist." + | exit 1 + |fi""".stripMargin + } else if (direction == Input) { s"""if [ ! -z "$$${param.VIASH_PAR}" ]; then | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' | set -f @@ -469,11 +479,11 @@ object BashWrapper { | done | set +f |fi""".stripMargin - } else { - s"""if [ ! -z "$$${param.VIASH_PAR}" ] && [ ! -e "$$${param.VIASH_PAR}" ]; then - | ViashError "$direction file '$$${param.VIASH_PAR}' does not exist." - | exit 1 - |fi""".stripMargin + } else { // multiple: true, direction: output expects arguments in the form of "output_*.txt" + s"""if [ ! -z "$$${param.VIASH_PAR}" ] && ! compgen -G "$$${param.VIASH_PAR}" > /dev/null; then + | ViashError "$direction file '$$${param.VIASH_PAR}' does not exist." + | exit 1 + |fi""".stripMargin } }.mkString("\n") } @@ -492,7 +502,7 @@ object BashWrapper { } else { "\n# create parent directories of output files, if so desired\n" + createParentFiles.map { param => - if (param.multiple) { + if (param.multiple && param.direction == Input) { s"""if [ ! -z "$$${param.VIASH_PAR}" ]; then | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' | set -f @@ -515,7 +525,8 @@ object BashWrapper { // construct type checks def typeMinMaxCheck[T](param: Argument[T], regex: String, min: Option[T] = None, max: Option[T] = None) = { val typeWithArticle = param match { - case i: IntegerArgument => "an " + param.`type` + case _: FileArgument if param.multiple && param.direction == Output => "a path containing a wildcard, e.g. 'output_*.txt'" + case _: IntegerArgument => "an " + param.`type` case _ => "a " + param.`type` } @@ -583,7 +594,7 @@ object BashWrapper { } param match { - case param if param.multiple => + case param if param.multiple && param.direction == Input => val checkStart = s"""if [ -n "$$${param.VIASH_PAR}" ]; then | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' @@ -624,17 +635,18 @@ object BashWrapper { Some(typeMinMaxCheck(dO, "^[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?$", dO.min, dO.max)) case bo: BooleanArgumentBase => Some(typeMinMaxCheck(bo, "^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$")) + case fo: FileArgument if fo.multiple && fo.direction == Output => + Some(typeMinMaxCheck(fo, "\\*")) case _ => None } }.mkString("\n") } - def checkChoices[T](param: Argument[T], allowedChoices: List[T]) = { val allowedChoicesString = allowedChoices.mkString(param.multiple_sep.toString) param match { - case _ if param.multiple => + case _ if param.multiple && param.direction == Input => s"""if [ ! -z "$$${param.VIASH_PAR}" ]; then | ${param.VIASH_PAR}_CHOICES=("$allowedChoicesString") | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' @@ -777,7 +789,7 @@ object BashWrapper { case param => val flag = if (param.flags == "") "" else " " + param.name - if (param.multiple) { + if (param.multiple && param.direction == Input) { s""" |if [ ! -z "$$${param.VIASH_PAR}" ]; then | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' diff --git a/src/test/resources/testbash/code.sh b/src/test/resources/testbash/code.sh index 0b8d8468c..d341a11e4 100755 --- a/src/test/resources/testbash/code.sh +++ b/src/test/resources/testbash/code.sh @@ -9,6 +9,7 @@ par_truth="true" par_falsehood="false" par_reality="" par_output="output.txt" +par_multiple_output="output*.txt" par_log="log.txt" par_optional="help" par_optional_with_default="me" @@ -55,5 +56,17 @@ output "optional: |$par_optional|" output "optional_with_default: |$par_optional_with_default|" output "multiple: |$par_multiple|" output "multiple_pos: |$par_multiple_pos|" - +output "multiple_output: |$par_multiple_output|" output "meta_resources_dir: |$meta_resources_dir|" + +multiple_output_i=0 +if [ ! -z "$par_multiple_output" ]; then + multiple_output_file=$(echo $par_multiple_output | sed "s/\*/$multiple_output_i/") + cp "$par_input" "$multiple_output_file" + multiple_output_i=$((multiple_output_i+1)) + + multiple_output_file=$(echo $par_multiple_output | sed "s/\*/$multiple_output_i/") + cp "$par_input" "$multiple_output_file" + multiple_output_i=$((multiple_output_i+1)) +fi + diff --git a/src/test/resources/testbash/config.vsh.yaml b/src/test/resources/testbash/config.vsh.yaml index c29fb6c13..767ce505a 100755 --- a/src/test/resources/testbash/config.vsh.yaml +++ b/src/test/resources/testbash/config.vsh.yaml @@ -72,6 +72,10 @@ functionality: - name: "multiple_pos" type: string multiple: true + - name: "--multiple_output" + type: file + multiple: true + direction: output resources: - type: bash_script path: ./code.sh diff --git a/src/test/resources/testbash/docker_options/code_multiple_output.sh b/src/test/resources/testbash/docker_options/code_multiple_output.sh index 02be284e3..656dbb728 100755 --- a/src/test/resources/testbash/docker_options/code_multiple_output.sh +++ b/src/test/resources/testbash/docker_options/code_multiple_output.sh @@ -9,60 +9,17 @@ par_truth="true" par_falsehood="false" par_reality="" par_output="output.txt" +par_multiple_output="output_*.txt" par_log="log.txt" par_optional="help" par_optional_with_default="me" meta_resources_dir="." # VIASH END -set -e +par_out_1=`echo "$par_multiple_output" | sed 's/\*/1/g'` +par_out_2=`echo "$par_multiple_output" | sed 's/\*/2/g'` +par_out_3=`echo "$par_multiple_output" | sed 's/\*/3/g'` -function log { - if [ -z "$par_log" ]; then - echo "$@" - else - echo "$@" >> $par_log - fi -} -function output { - if [ -z "$par_output" ]; then - echo "$@" - else - echo "$@" >> $par_output - - if [ ! -z "$par_output_pos" ]; then - IFS=":" - for var in $par_output_pos; do - unset IFS - echo "$@" >> $var - done - fi - - fi -} - -log "INFO: Parsed input arguments." - -if [ -z "$par_output" ]; then - log "INFO: Printing output to console" -else - log "INFO: Writing output to file" -fi - -output "input: |$par_input|" -output "real_number: |$par_real_number|" -output "whole_number: |$par_whole_number|" -output "s: |$par_s|" -output "truth: |$par_truth|" -output "falsehood: |$par_falsehood|" -output "reality: |$par_reality|" -output "output: |$par_output|" -output "log: |$par_log|" -output "optional: |$par_optional|" -output "optional_with_default: |$par_optional_with_default|" -output "meta_resources_dir: |$meta_resources_dir|" -INPUT=`head -1 "$par_input"` -output "head of input: |$INPUT|" -RESOURCE=`head -1 "$meta_resources_dir/resource1.txt"` -output "head of resource1: |$RESOURCE|" -output "output_pos: |$par_output_pos|" +echo "input: |$par_input|" > $par_out_1 +echo "real_number: |$par_real_number|" > $par_out_2 +echo "whole_number: |$par_whole_number|" > $par_out_3 diff --git a/src/test/resources/testbash/tests/check_outputs.sh b/src/test/resources/testbash/tests/check_outputs.sh index ece0cd0c6..a146e3543 100755 --- a/src/test/resources/testbash/tests/check_outputs.sh +++ b/src/test/resources/testbash/tests/check_outputs.sh @@ -14,7 +14,8 @@ echo ">>> Checking whether output is correct" --output ./output.txt --log ./log.txt \ --multiple one --multiple=two \ e f \ - --long_number 112589990684262400 + --long_number 112589990684262400 \ + --multiple_output './output_*.txt' [[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 grep -q 'input: |NOTICE|' output.txt @@ -31,5 +32,6 @@ grep -q 'optional: |foo|' output.txt grep -q 'optional_with_default: |bar|' output.txt grep -q 'multiple: |one:two|' output.txt grep -q 'multiple_pos: |a:b:c:d:e:f|' output.txt +grep -q 'multiple_output: |.*/output_\*.txt|' output.txt echo ">>> Test finished successfully" diff --git a/src/test/resources/testnextflowvdsl3/src/multiple_output/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/multiple_output/config.vsh.yaml new file mode 100644 index 000000000..94d0eeec5 --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/multiple_output/config.vsh.yaml @@ -0,0 +1,24 @@ +functionality: + name: multiple_output + arguments: + - name: "--input" + type: file + multiple: true + required: true + example: input.txt + multiple_sep: ";" + - name: "--output" + type: file + required: true + direction: output + multiple: true + multiple_sep: ";" + example: output_*.txt + resources: + - type: bash_script + path: script.sh +platforms: + - type: native + - type: docker + image: nextflow/bash:latest + - type: nextflow diff --git a/src/test/resources/testnextflowvdsl3/src/multiple_output/script.sh b/src/test/resources/testnextflowvdsl3/src/multiple_output/script.sh new file mode 100644 index 000000000..c9ffcea7a --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/multiple_output/script.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +## VIASH START +par_input='resources/lines3.txt;resources/lines5.txt' +par_output='output_*.txt' +## VIASH END + +output_i=0 + +if [ ! -z "$par_input" ]; then + IFS=";" + for var in $par_input; do + unset IFS + output=$(echo "$par_output" | sed "s/\*/$output_i/g") + cp "$var" "$output" + output_i=$((output_i+1)) + done +fi diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf b/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf index d16051213..e3a649ad2 100644 --- a/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf @@ -6,7 +6,7 @@ workflow base { ch = Channel.fromList(0..1000) | map { num -> // create temporary file - file = tempFile() + def file = tempFile() file.write("num: $num") ["num$num", [ num: num, file: file ]] diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/nested/main.nf b/src/test/resources/testnextflowvdsl3/src/test_wfs/nested/main.nf index 2ba19788d..96e810b5d 100644 --- a/src/test/resources/testnextflowvdsl3/src/test_wfs/nested/main.nf +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/nested/main.nf @@ -6,7 +6,7 @@ workflow base { ch = Channel.fromList(0..1000) | map { num -> // create temporary file - file = tempFile() + def file = tempFile() file.write("num: $num") ["num$num", [ file: file ], ["num": num]] diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf b/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf index ce752a546..9eca5abb4 100644 --- a/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf @@ -15,7 +15,7 @@ workflow base { | map { num -> // create temporary file - file = tempFile() + def file = tempFile() file.write("num: $num") ["num$num", [ num: num, file: file ]] @@ -26,7 +26,6 @@ workflow base { toState: ["step1_output": "output"] ) - | step1.run( key: "step1bis", fromState: { id, state -> diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala index a5a1f3152..cdcb27fb7 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala @@ -30,10 +30,7 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll w ) val multipleOutputsMods = List( - """del(.functionality.argument_groups[.name == "Arguments"].arguments[.name == "--multiple"])""", - """del(.functionality.argument_groups[.name == "Arguments"].arguments[.name == "multiple_pos"])""", - """.functionality.argument_groups[.name == "Arguments"].arguments[.name == "--output"].multiple := true""", - """.functionality.argument_groups[.name == "Arguments"].arguments += {name: "output_pos", type: "file", direction: "output", multiple: true}""", + """.functionality.argument_groups[.name == "Arguments"].arguments[.name == "--output"].required := false""", """.functionality.resources[.type == "bash_script"].path := "docker_options/code_multiple_output.sh"""", ) @@ -61,14 +58,12 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll w assert(localExecutable.canExecute) // run the script - val output = Paths.get(tempFolStr, "output_" + dockerId + ".txt").toFile - val output2 = Paths.get(tempFolStr, "output_" + dockerId +"_2.txt").toFile - val output3 = Paths.get(tempFolStr, "output_" + dockerId +"_3.txt").toFile + val pattern = f"$tempFolStr/output_${dockerId}_*.txt" val outputParams = amount match { - case 1 => Seq("--output", output.getPath) - case 2 => Seq("--output", output.getPath, "--output2", output2.getPath) - case 3 => Seq("--output", output.getPath, output2.getPath, output3.getPath) + case 1 => Seq("--output", pattern.replace("*", "1")) + case 2 => Seq("--output", pattern.replace("*", "1"), "--output2", pattern.replace("*", "2")) + case 3 => Seq("--multiple_output", pattern) } Exec.run( @@ -81,9 +76,12 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll w ) ++ outputParams ) - val outputList = List(output, output2, output3).take(amount) - outputList.foreach(output => assert(output.exists())) - outputList.map(file => Files.getOwner(file.toPath).toString) + // find files based on the pattern + import scala.jdk.CollectionConverters._ + val foundFiles = Files.list(temporaryFolder).iterator().asScala.toList + val outputFiles = foundFiles.filter(_.toString.matches(pattern.replace("*", ".*"))) + assert(outputFiles.length == amount, s"Expected $amount files, got ${outputFiles.length}.\nDirectory contents: ${foundFiles.mkString(", ")}") + outputFiles.map(file => Files.getOwner(file).toString) } test("Test default behaviour when chown is not specified", DockerTest) { diff --git a/src/test/scala/io/viash/e2e/build/DockerSuite.scala b/src/test/scala/io/viash/e2e/build/DockerSuite.scala index bb05704e2..d1402ef6d 100644 --- a/src/test/scala/io/viash/e2e/build/DockerSuite.scala +++ b/src/test/scala/io/viash/e2e/build/DockerSuite.scala @@ -111,7 +111,7 @@ class DockerSuite extends AnyFunSuite with BeforeAndAfterAll { assert(outputLines.contains("""optional_with_default: |bar|""")) assert(outputLines.contains("""multiple: |foo:bar|""")) assert(outputLines.contains("""multiple_pos: |a:b:c:d:e:f|""")) - val regex = s"""meta_resources_dir: \\|.*${temporaryFolder}/\\|""".r + val regex = s"""meta_resources_dir: \\|.*${temporaryFolder}\\|""".r assert(regex.findFirstIn(outputLines).isDefined) } finally { outputSrc.close() @@ -148,7 +148,7 @@ class DockerSuite extends AnyFunSuite with BeforeAndAfterAll { assert(stdout.contains("""optional_with_default: |The default value.|""")) assert(stdout.contains("""multiple: ||""")) assert(stdout.contains("""multiple_pos: ||""")) - val regex = s"""meta_resources_dir: \\|/viash_automount.*$temporaryFolder/\\|""".r + val regex = s"""meta_resources_dir: \\|/viash_automount.*$temporaryFolder\\|""".r assert(regex.findFirstIn(stdout).isDefined) assert(stdout.contains("INFO: Parsed input arguments")) @@ -157,4 +157,4 @@ class DockerSuite extends AnyFunSuite with BeforeAndAfterAll { override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) } -} \ No newline at end of file +} diff --git a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala index 269d7ffcd..8c3eabdf1 100644 --- a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala @@ -109,6 +109,96 @@ class MainNSBuildNativeSuite extends AnyFunSuite with BeforeAndAfterAll{ } } + test("Check uniqueness of component names, same name, different namespace") { + val compStr = + """functionality: + | name: comp + | namespace: %s + |""".stripMargin + val tempSrcDir = IO.makeTemp("viash_ns_build_check_uniqueness_src") + IO.write(compStr.format("ns1"), tempSrcDir.resolve("config1.vsh.yaml")) + IO.write(compStr.format("ns2"), tempSrcDir.resolve("config2.vsh.yaml")) + + val tempTargetDir = IO.makeTemp("viash_ns_build_check_uniqueness_target") + + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "build", + "-s", tempSrcDir.toString(), + "-t", tempTargetDir.toString() + ) + + assert(exitCode == 0) + assert(stderr.contains("All 2 configs built successfully")) + } + + test("Check uniqueness of component names, different name, same namespace") { + val compStr = + """functionality: + | name: %s + | namespace: ns + |""".stripMargin + val tempSrcDir = IO.makeTemp("viash_ns_build_check_uniqueness_src") + IO.write(compStr.format("comp1"), tempSrcDir.resolve("config1.vsh.yaml")) + IO.write(compStr.format("comp2"), tempSrcDir.resolve("config2.vsh.yaml")) + + val tempTargetDir = IO.makeTemp("viash_ns_build_check_uniqueness_target") + + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "build", + "-s", tempSrcDir.toString(), + "-t", tempTargetDir.toString() + ) + + assert(exitCode == 0) + assert(stderr.contains("All 2 configs built successfully")) + } + + test("Check uniqueness of component names, same name, same namespace") { + val compStr = + """functionality: + | name: %s + | namespace: ns + |""".stripMargin + val tempSrcDir = IO.makeTemp("viash_ns_build_check_uniqueness_src") + IO.write(compStr.format("comp"), tempSrcDir.resolve("config1.vsh.yaml")) + IO.write(compStr.format("comp"), tempSrcDir.resolve("config2.vsh.yaml")) + + val tempTargetDir = IO.makeTemp("viash_ns_build_check_uniqueness_target") + + val testOutput = TestHelper.testMainException2[RuntimeException]( + "ns", "build", + "-s", tempSrcDir.toString(), + "-t", tempTargetDir.toString() + ) + + assert(!testOutput.error.contains("All 2 configs built successfully")) + assert(testOutput.exceptionText.contains("Duplicate component name found: ns/comp")) + } + + test("Check uniqueness of component names, same name, same namespace - multiple duplicates") { + val compStr = + """functionality: + | name: %s + | namespace: ns + |""".stripMargin + val tempSrcDir = IO.makeTemp("viash_ns_build_check_uniqueness_src") + IO.write(compStr.format("comp1"), tempSrcDir.resolve("config1.vsh.yaml")) + IO.write(compStr.format("comp1"), tempSrcDir.resolve("config2.vsh.yaml")) + IO.write(compStr.format("comp2"), tempSrcDir.resolve("config3.vsh.yaml")) + IO.write(compStr.format("comp2"), tempSrcDir.resolve("config4.vsh.yaml")) + + val tempTargetDir = IO.makeTemp("viash_ns_build_check_uniqueness_target") + + val testOutput = TestHelper.testMainException2[RuntimeException]( + "ns", "build", + "-s", tempSrcDir.toString(), + "-t", tempTargetDir.toString() + ) + + assert(!testOutput.error.contains("All 2 configs built successfully")) + assert(testOutput.exceptionText.contains("Duplicate component names found: ns/comp1, ns/comp2")) + } + override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) } diff --git a/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala index 489ab1ec2..84a84bd31 100644 --- a/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala @@ -96,6 +96,39 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error") } + + test("Check namespace test output with working dir message using --dry_run") { + val (stdout, stderr, _) = TestHelper.testMainWithStdErr( + "ns", "test", + "--src", nsPath, + "--keep", "true", + "--dry_run" + ) + + // Test inclusion of a header + val regexHeader = raw"^\s*namespace\s*functionality\s*platform\s*test_name\s*exit_code\s*duration\s*result".r + assert(regexHeader.findFirstIn(stdout).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n${stdout}") + + val regexWdir = raw"The working directory for the namespace tests is [\w/]+[\r\n]{1,2}".r + assert(regexWdir.findFirstIn(stderr).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n${stderr}") + + for ( + (component, steps) <- components; + (step, resultPattern) <- steps + ) { + // In this mode, a test script can never return an ERROR + val updtResultPattern = if (resultPattern == raw"\s*1\s*\d+\s*ERROR") { + raw"\s*0\s*\d+\s*SUCCESS" + } else { + resultPattern + } + val regex = s"""testns\\s*$component\\s*native\\s*$step$updtResultPattern""".r + assert(regex.findFirstIn(stdout).isDefined, s"\nRegex: '${regex.toString}'; text: \n${stdout}") + } + + val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r + assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error") + } test("Check namespace test output with tsv option") { val log = Paths.get(tempFolStr, "log.tsv").toFile @@ -194,6 +227,37 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { } } + test("Check namespace test output with deterministic working directory") { + // create a new unique temporary folder + val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + val tempFolStr = temporaryFolder.getFileName().toString() + + val (stdout, stderr, _) = TestHelper.testMainWithStdErr( + "ns", "test", + "--src", nsPath, + "--deterministic_working_directory", tempFolStr + ) + + assert(stderr.contains(s"The working directory for the namespace tests is ${temporaryFolder}")) + + // Test inclusion of a header + val regexHeader = raw"^\s*namespace\s*functionality\s*platform\s*test_name\s*exit_code\s*duration\s*result".r + assert(regexHeader.findFirstIn(stdout).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n$stdout") + + for ( + (component, steps) <- components; + (step, resultPattern) <- steps + ) { + val regex = s"""testns\\s*$component\\s*native\\s*$step$resultPattern""".r + assert(regex.findFirstIn(stdout).isDefined, s"\nRegex: '${regex.toString}'; text: \n$stdout") + } + + val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r + assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error") + + assert(stderr.contains("The status of the component 'ns_power' is set to deprecated.")) + } + override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) } diff --git a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala index 0558ccff1..76fc8b269 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala @@ -228,6 +228,21 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { checkTempDirAndRemove(testText, true) } + test("Check output in case --dry_run is specified") { + val testText = TestHelper.testMain( + "test", + "-p", "native", + "--dry_run", + configFile + ) + assert(testText.contains("Running tests in temporary directory: ")) + assert(testText.contains("Running dummy test script")) + assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!")) + assert(testText.contains("Cleaning up temporary directory")) + + checkTempDirAndRemove(testText, false) + } + test("Check output in case --keep false is specified") { val testText = TestHelper.testMain( "test", @@ -311,6 +326,64 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { assert(testOutput.output.isEmpty) } + test("Check standard test output with deterministic build folder that exists but is empty") { + // create a new unique temporary folder + val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + val tempFolStr = temporaryFolder.getFileName().toString() + + val testText = TestHelper.testMain( + "test", + "-p", "native", + "--deterministic_working_directory", tempFolStr, + configFile + ) + + assert(testText.contains(s"Running tests in temporary directory: '${temporaryFolder}'")) + assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!")) + assert(testText.contains("Cleaning up temporary directory")) + + checkTempDirAndRemove(testText, false, tempFolStr) + } + + test("Check standard test output with deterministic build folder that doesn't exist yet") { + // create a new unique temporary folder to know what folder name to use + val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + temporaryFolder.toFile().delete() + val tempFolStr = temporaryFolder.getFileName().toString() + + val testText = TestHelper.testMain( + "test", + "-p", "native", + "--deterministic_working_directory", tempFolStr, + configFile + ) + + assert(testText.contains(s"Running tests in temporary directory: '${temporaryFolder}'")) + assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!")) + assert(testText.contains("Cleaning up temporary directory")) + + checkTempDirAndRemove(testText, false, tempFolStr) + } + + test("Check standard test output with deterministic build folder that isn't empty") { + // create a new unique temporary folder to know what folder name to use + val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + IO.write("foo", temporaryFolder.resolve("foo.txt")) + val tempFolStr = temporaryFolder.getFileName().toString() + + val testOutput = TestHelper.testMainException2[RuntimeException]( + "test", + "-p", "native", + "--deterministic_working_directory", tempFolStr, + configFile + ) + + assert(testOutput.exceptionText.contains(s"${temporaryFolder} already exists and is not empty.")) + + assert(testOutput.output.isEmpty()) + assert(testOutput.error.isEmpty()) + } + /** * Searches the output generated by Main.main() during tests for the temporary directory name and verifies if it still exists or not. * If directory was expected to be present and actually is present, it will be removed. @@ -318,7 +391,7 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { * @param expectDirectoryExists expect the directory to be present or not * @return */ - def checkTempDirAndRemove(testText: String, expectDirectoryExists: Boolean): Unit = { + def checkTempDirAndRemove(testText: String, expectDirectoryExists: Boolean, testDirName: String = "viash_test_testbash"): Unit = { // Get temporary directory val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r @@ -327,7 +400,7 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { case _ => "" } - assert(tempPath.contains(s"${IO.tempDir}/viash_test_testbash")) + assert(tempPath.contains(s"${IO.tempDir}/$testDirName")) val tempFolder = new Directory(Paths.get(tempPath).toFile) diff --git a/src/test/scala/io/viash/functionality/MainScriptTest.scala b/src/test/scala/io/viash/functionality/MainScriptTest.scala new file mode 100644 index 000000000..ab485abcc --- /dev/null +++ b/src/test/scala/io/viash/functionality/MainScriptTest.scala @@ -0,0 +1,108 @@ +package io.viash.functionality + +import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger +import io.viash.functionality.resources.{BashScript, PlainFile} + +class MainScriptTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) + + test("Single script") { + val resources = List(BashScript(path = Some("foo.sh"))) + val fun = Functionality("fun", resources = resources) + + val mainScript = fun.mainScript + val additionalResources = fun.additionalResources + + assert(mainScript.isDefined) + assert(mainScript.get.path == Some("foo.sh")) + assert(additionalResources.isEmpty) + } + + test("Multiple scripts") { + val resources = List( + BashScript(path = Some("foo.sh")), + BashScript(path = Some("bar.sh")), + BashScript(path = Some("baz.sh")), + ) + val fun = Functionality("fun", resources = resources) + + val mainScript = fun.mainScript + val additionalResources = fun.additionalResources + + assert(mainScript.isDefined) + assert(mainScript.get.path == Some("foo.sh")) + assert(additionalResources.length == 2) + } + + test("Single text file") { + val resources = List(PlainFile(path = Some("foo.txt"))) + val fun = Functionality("fun", resources = resources) + + val mainScript = fun.mainScript + val additionalResources = fun.additionalResources + + assert(mainScript.isEmpty) + assert(additionalResources.length == 1) + assert(additionalResources.head.path == Some("foo.txt")) + } + + test("Multiple text files") { + val resources = List( + PlainFile(path = Some("foo.txt")), + PlainFile(path = Some("bar.txt")), + PlainFile(path = Some("baz.txt")), + ) + val fun = Functionality("fun", resources = resources) + + val mainScript = fun.mainScript + val additionalResources = fun.additionalResources + + assert(mainScript.isEmpty) + assert(additionalResources.length == 3) + } + + test("Mixed Script and text files, first is script") { + val resources = List( + BashScript(path = Some("foo.sh")), + PlainFile(path = Some("bar.txt")), + BashScript(path = Some("baz.sh")), + ) + + val fun = Functionality("fun", resources = resources) + + val mainScript = fun.mainScript + val additionalResources = fun.additionalResources + + assert(mainScript.isDefined) + assert(mainScript.get.path == Some("foo.sh")) + assert(additionalResources.length == 2) + } + + test("Mixed Script and text files, first is text file") { + val resources = List( + PlainFile(path = Some("foo.txt")), + BashScript(path = Some("bar.sh")), + PlainFile(path = Some("baz.txt")), + ) + + val fun = Functionality("fun", resources = resources) + + val mainScript = fun.mainScript + val additionalResources = fun.additionalResources + + assert(mainScript.isEmpty) + assert(additionalResources.length == 3) + } + + test("No resources") { + val resources = List.empty + val fun = Functionality("fun", resources = resources) + + val mainScript = fun.mainScript + val additionalResources = fun.additionalResources + + assert(mainScript.isEmpty) + assert(additionalResources.isEmpty) + } +} \ No newline at end of file diff --git a/src/test/scala/io/viash/helpers/IOTest.scala b/src/test/scala/io/viash/helpers/IOTest.scala index 97bde729c..0159d3007 100644 --- a/src/test/scala/io/viash/helpers/IOTest.scala +++ b/src/test/scala/io/viash/helpers/IOTest.scala @@ -22,11 +22,62 @@ class IOTest extends AnyFunSuite with BeforeAndAfter { test("makeTemp and deleteRecursively") { val temp = IO.makeTemp("foo") assert(Files.exists(temp) && Files.isDirectory(temp)) + assert(temp.toString.matches(".*foo[\\w]+"), "Temporary directory name should be randomized, strategy can differ between platforms.") IO.deleteRecursively(temp) assert(!Files.exists(temp)) } + test("makeTemp with addRandomized disabled, folder exists") { + val tempDir = IO.makeTemp("foo") + val tempDirStr = tempDir.getFileName().toString() + val newTemp = IO.makeTemp(tempDirStr, addRandomized = false) + assert(Files.exists(newTemp) && Files.isDirectory(newTemp)) + + assert(newTemp == tempDir) + IO.deleteRecursively(newTemp) + } + + test("makeTemp with addRandomized disabled, folder doesn't exist") { + val tempDir = IO.makeTemp("foo") + val tempDirStr = tempDir.getFileName().toString() + IO.deleteRecursively(tempDir) + val newTemp = IO.makeTemp(tempDirStr, addRandomized = false) + assert(Files.exists(newTemp) && Files.isDirectory(newTemp)) + + assert(newTemp == tempDir) + IO.deleteRecursively(newTemp) + } + + test("makeTemp with addRandomized disabled, folder exists but not empty") { + val tempDir = IO.makeTemp("foo") + val tempDirStr = tempDir.getFileName().toString() + IO.write("foo", tempDir.resolve("foo.txt")) + + val caught = intercept[RuntimeException] { + IO.makeTemp(tempDirStr, addRandomized = false) + } + + assert(caught.getMessage().contains(s"Temporary directory $tempDir already exists and is not empty.")) + + IO.deleteRecursively(tempDir) + } + + test("makeTemp with addRandomized disabled, folder exists as a file") { + val tempDir = IO.makeTemp("foo") + val tempDirStr = tempDir.getFileName().toString() + IO.deleteRecursively(tempDir) + IO.write("foo", tempDir) + + val caught = intercept[RuntimeException] { + IO.makeTemp(tempDirStr, addRandomized = false) + } + + assert(caught.getMessage().contains(s"Temporary directory $tempDir already exists as a file.")) + + IO.deleteRecursively(tempDir) + } + test("uri with path") { val uri = IO.uri("test.txt") assert(uri.isInstanceOf[URI]) diff --git a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala index 02673886c..cf0b689da 100644 --- a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala @@ -14,6 +14,14 @@ import java.io.UncheckedIOException import NextflowTestHelper._ +/** + * This test suite contains tests related to the NextflowScript class. + * + * All workflows tested in this suite should be self-contained in that + * they execute tests and do assertions within the same workflow. If + * the test fails, the workflow should print a helpful error message and + * exit with a non-zero exit code. + */ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) // temporary folder to work in @@ -145,8 +153,15 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { ) assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - assert(stdOut.contains("base:step1_alias:proc")) - assert(stdOut.contains("base:step1:proc")) + // Note: Nextflow 23.04 changed the way processes are printed in the log + // original: + // [info] [30/b6eaaf] process > alias:base:step1:processWf:... [100%] 1 of 1 ✔ + // [info] [bf/166913] process > alias:base:step1_alias:proc... [100%] 1 of 1 ✔ + // since nextflow 23.04: + // [info] [31/1c0834] ali…cessWf:step1_process (one) | 1 of 1 ✔ + // [info] [f1/e92549] ali…:step1_alias_process (one) | 1 of 1 ✔ + assert(stdOut.contains(":step1_alias:proc") || stdOut.contains(":step1_alias_process")) + assert(stdOut.contains(":step1:proc") || stdOut.contains(":step1_process")) assert(!stdOut.contains("Key for module 'step1' is duplicated")) } @@ -202,7 +217,6 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut1\nStd error:\n$stdErr1") } - override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) } diff --git a/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala b/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala index 6ba262c72..c664f0b0e 100644 --- a/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala @@ -8,6 +8,7 @@ import java.io.UncheckedIOException import java.io.File import java.nio.file.{Files, Path, Paths} import scala.io.Source +import scala.util.Using import io.viash.helpers.{IO, Logger} import io.viash.{DockerTest, NextflowTest, TestHelper} @@ -15,6 +16,11 @@ import java.nio.charset.StandardCharsets import NextflowTestHelper._ +/** + * Test suite for VDSL3 components as standalone Nextflow workflows. + * Since these tests run workflows from the CLI, we need to check the + * generated output inside the test suite. + */ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) // temporary folder to work in @@ -163,6 +169,48 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { } } + + test("Run multiple output test", NextflowTest) { + val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + mainScript = "target/nextflow/multiple_output/main.nf", + args = List( + "--id", "foo", + "--input", "resources/lines*.txt", + "--publish_dir", "multipleOutput" + ), + cwd = tempFolFile + ) + + assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + + val expectedFiles = + Map( + "state" -> "state.yaml", + "output_0" -> "output_0.txt", + "output_1" -> "output_1.txt" + ).map{ case (id, suffix) => + val path = temporaryFolder.resolve("multipleOutput/foo.multiple_output." + suffix) + (id, path) + } + + // check if files exist + for ((id, path) <- expectedFiles) { + assert(Files.exists(path), s"File '$id' at path '$path' does not exist") + } + + // check state content + Using(Source.fromFile(expectedFiles("state").toFile())) { reader => + val stateTxt = reader.getLines().mkString("\n") + val expectedState = """\ + |id: foo + |output: + |- !file 'foo.multiple_output.output_0.txt' + |- !file 'foo.multiple_output.output_1.txt' + |""".stripMargin + assert(stateTxt == expectedState) + } + } + override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) }