Merge pull request #692 from viash-io/develop_0_8

Develop 0 8

.github/workflows/sbt_test.yml

@@ -42,12 +42,19 @@ jobs:
           processx
           testthat
 
-    - name: Set up sbt
+    - name: Set up java & sbt
       uses: actions/setup-java@v4
       with:
         distribution: temurin
         java-version: ${{ matrix.java.ver }}
 
+    - name: Set up sbt specifically on macOS on arm64 if needed
+      if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64'}}
+      run: |
+        if ! command -v sbt &> /dev/null; then
+          brew install sbt
+        fi
+
     - name: Set up Scala
       run: |
         if [[ "${{ matrix.config.os }}" =~ ^macos.*$ ]]; then

CHANGELOG.md

@@ -1,10 +1,44 @@
+# Viash 0.8.6 (2024-04-26): Bug fixes and improvements for CI
+
+Fix some issues in some edge cases.
+Add options for testing in a CI environment. Given that these options are not meant for general use, they are hidden from the help message.
+Some improvements are made to run in Nextflow Fusion.
+
+## DOCUMENTATION
+
+* `docker setup strategy`: Fix inconsistencies in the documentation (PR #657).
+
+* `repositories`: Fix `uri` -> `repo` in the repositories documentation (PR #682).
+
+## NEW FUNCTIONALITY
+
+* `viash test` and `viash ns test`: Add a hidden `--dry_run` option to build the tests without executing them (PR #676).
+
+* `viash test` and `viash ns test`: Add a hidden `--deterministic_working directory` argument to use a fixed directory path (PR #683).
+
+* `component names`: Verify that component namespace and name combinations are unique (PR #685).
+
+## BUG FIXES
+
+* `NextflowPlatform`: Fix publishing state for output arguments with `multiple: true` (#638, PR #639). 
+
+* `Executable`: Check whether a multiple output file argument contains a wildcard (PR #639).
+
+* `NextflowPlatform`: Fix a possible cause of concurrency issues (PR #669).
+
+* `Resources`: Fix an issue where if the first resource is not a script, the resource is silently dropped (PR #670).
+
+* `Docker automount`: Prevent adding a trailing slash to an automounted folder (PR #673).
+
+* `NextflowPlatform`: Change the at-runtime generated nextflow process from an in-memory to an on-disk temporary file, which should cause less issues with Nextflow Fusion (PR #681).
+
 # Viash 0.8.5 (2024-02-21): Bug fixes and documentation improvements
 
 Fix a bug when building a test docker container which requires a test resource. Additional improvements for the website documentation and support for the latest version of Nextflow are added.
 
 ## BUG FIXES
 
-* `nextflow runner`: Fix an issue with current nextflow-latest (24.01.0-edge) where our supporting library passes a GString instead of a String and results in a type mismatch (PR #640).
+* `NextflowPlatform`: Fix an issue with current nextflow-latest (24.01.0-edge) where our supporting library passes a GString instead of a String and results in a type mismatch (PR #640).
 
 * `test resources`: Make non-script test resources available during building of a docker container for `viash test` (PR #652).
 

build.sbt

@@ -1,6 +1,6 @@
 name := "viash"
 
-version := "0.8.5"
+version := "0.8.6"
 
 scalaVersion := "2.13.10"
 

src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh

@@ -15,7 +15,11 @@ function ViashAutodetectMount {
     base_name=`basename "$abs_path"`
   fi
   mount_target="/viash_automount$mount_source"
-  echo "$mount_target/$base_name"
+  if [ -z "$base_name" ]; then
+    echo "$mount_target"
+  else
+    echo "$mount_target/$base_name"
+  fi
 }
 function ViashAutodetectMountArg {
   abs_path=$(ViashAbsolutePath "$1")

src/main/resources/io/viash/platforms/nextflow/DataflowHelper.nf

@@ -52,16 +52,16 @@ def setWorkflowArguments(Map args) {
         // determine splitargs
         def splitArgs = args.
           collectEntries{procKey, dataKeys -> 
-          // dataKeys is a map but could also be a list
-          newSplitData = dataKeys
-            .collectEntries{ val ->
-              newKey = val instanceof String ? val : val.key
-              origKey = val instanceof String ? val : val.value
-              [ newKey, data[origKey] ]
-            }
-            .findAll{it.value}
-          [procKey, newSplitData]
-        }
+            // dataKeys is a map but could also be a list
+            def newSplitData = dataKeys
+              .collectEntries{ val ->
+                newKey = val instanceof String ? val : val.key
+                origKey = val instanceof String ? val : val.value
+                [ newKey, data[origKey] ]
+              }
+              .findAll{it.value}
+            [procKey, newSplitData]
+          }
 
         // return output
         [ id, newData, splitArgs] + passthrough

src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf

@@ -84,7 +84,7 @@ def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) {
           .findAll { it.type == "file" && it.direction == "output" }
           .indexed()
           .collectEntries{ index, par ->
-            out = output[index + 1]
+            def out = output[index + 1]
             // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678)
             if (!out instanceof List || out.size() <= 1) {
               if (par.multiple) {
@@ -210,7 +210,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   def inputFileExports = meta.config.functionality.allArguments
     .findAll { it.type == "file" && it.direction.toLowerCase() == "input" }
     .collect { par ->
-      viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})"
+      def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})"
       "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}"
     }
 
@@ -266,7 +266,6 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   |  .join("\\n")
   |\"\"\"
   |# meta exports
-  |# export VIASH_META_RESOURCES_DIR="\${resourcesDir.toRealPath().toAbsolutePath()}"
   |export VIASH_META_RESOURCES_DIR="\${resourcesDir}"
   |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}"
   |export VIASH_META_FUNCTIONALITY_NAME="${meta.config.functionality.name}"
@@ -308,19 +307,22 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
   //   println("######################\n$procStr\n######################")
   // }
 
-  // create runtime process
-  def ownerParams = new nextflow.script.ScriptBinding.ParamsMap()
-  def binding = new nextflow.script.ScriptBinding().setParams(ownerParams)
-  def module = new nextflow.script.IncludeDef.Module(name: procKey)
+  // write process to temp file
+  def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf")
+  addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) }
+  tempFile.text = procStr
+
+  // create process from temp file
+  def binding = new nextflow.script.ScriptBinding([:])
   def session = nextflow.Nextflow.getSession()
-  def scriptParser = new nextflow.script.ScriptParser(session)
+  def parser = new nextflow.script.ScriptParser(session)
     .setModule(true)
     .setBinding(binding)
-  scriptParser.scriptPath = scriptMeta.getScriptPath()
-  def moduleScript = scriptParser.runScript(procStr)
+  def moduleScript = parser.runScript(tempFile)
     .getScript()
 
   // register module in meta
+  def module = new nextflow.script.IncludeDef.Module(name: procKey)
   scriptMeta.addModule(moduleScript, module.name, module.alias)
 
   // retrieve and return process from meta

src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf

@@ -114,7 +114,7 @@ def _checkArgumentType(String stage, Map par, Object value, String errorIdentifi
     }
     expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
-    // cast to boolean if need ben
+    // cast to boolean if need be
     if (value instanceof String) {
       def valueLower = value.toLowerCase()
       if (valueLower == "true") {

src/main/resources/io/viash/platforms/nextflow/channel/_splitParams.nf

@@ -18,13 +18,13 @@ Map<String, Object> _splitParams(Map<String, Object> parValues, Map config){
     }
     if (parameterSettings.multiple) { // Check if parameter can accept multiple values
       if (parValue instanceof Collection) {
-          parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it }
+        parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it }
       } else if (parValue instanceof String) {
-          parValue = parValue.split(parameterSettings.multiple_sep)
+        parValue = parValue.split(parameterSettings.multiple_sep)
       } else if (parValue == null) {
-          parValue = []
+        parValue = []
       } else {
-          parValue = [ parValue ]
+        parValue = [ parValue ]
       }
       parValue = parValue.flatten()
     }

src/main/resources/io/viash/platforms/nextflow/channel/channelFromParams.nf

@@ -60,9 +60,9 @@ private List<Tuple2<String, Map<String, Object>>> _paramsToParamSets(Map params,
     // Process ID
     def id = tup[0] ?: globalID
 
-    if (workflow.stubRun) {
+    if (workflow.stubRun && !id) {
       // if stub run, explicitly add an id if missing
-      id = id ? id : "stub${index}"
+      id = "stub${index}"
     }
     assert id != null: "Each parameter set should have at least an 'id'"
 

src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf

@@ -77,6 +77,7 @@ def runComponents(Map args) {
         ? out_ch | map{tup ->
           def output = tup[1]
           def old_state = tup[2]
+          def new_state = null
           if (toState_ instanceof Map) {
             new_state = old_state + toState_.collectEntries{ key0, key1 ->
               [key0, output[key1]]

src/main/resources/io/viash/platforms/nextflow/functions/_stringIsAbsolutePath.nf

@@ -9,7 +9,7 @@
   * @return Whether the path is absolute, as a boolean.
   */
 def _stringIsAbsolutePath(path) {
-  _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/
+  def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/
 
   assert path instanceof String
   return _resolve_URL_PROTOCOL.matcher(path).matches()

src/main/resources/io/viash/platforms/nextflow/readwrite/readCsv.nf

@@ -38,7 +38,7 @@ def readCsv(file_path) {
         if (field == "") {
           return null
         }
-        m = removeQuote.matcher(field)
+        def m = removeQuote.matcher(field)
         if (m.find()) {
           return m.replaceFirst('$1')
         } else {

src/main/resources/io/viash/platforms/nextflow/states/findStates.nf

@@ -1,5 +1,4 @@
 def findStates(Map params, Map config) {
-  // TODO: do a deep clone of config
   def auto_config = deepClone(config)
   def auto_params = deepClone(params)
 

src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf

@@ -53,20 +53,20 @@ def publishStates(Map args) {
           def state_ = tup[1]
 
           // the input files and the target output filenames
-          def inputOutputFiles_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
-          def inputFiles_ = inputOutputFiles_[0]
-          def outputFiles_ = inputOutputFiles_[1]
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
 
           def yamlFilename = yamlTemplate_
             .replaceAll('\\$id', id_)
             .replaceAll('\\$key', key_)
 
-            // TODO: do the pathnames in state_ match up with the outputFiles_?
+            // TODO: do the pathnames in state_ match up with the outputFilenames_?
 
           // convert state to yaml blob
           def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFiles_]
+          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
         }
         | publishStatesProc
     emit: input_ch
@@ -99,12 +99,12 @@ process publishStatesProc {
       }
     }
   """
-  mkdir -p "\$(dirname '${yamlFile}')"
-  echo "Storing state as yaml"
-  echo '${yamlBlob}' > '${yamlFile}'
-  echo "Copying output files to destination folder"
-  ${copyCommands.join("\n  ")}
-  """
+mkdir -p "\$(dirname '${yamlFile}')"
+echo "Storing state as yaml"
+echo '${yamlBlob}' > '${yamlFile}'
+echo "Copying output files to destination folder"
+${copyCommands.join("\n  ")}
+"""
 }
 
 
@@ -133,9 +133,13 @@ def publishStatesByConfig(Map args) {
             .replaceAll('\\$key', key_)
           def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
 
-          // the processed state is a list of [key, value, srcPath, destPath] tuples, where
-          //   - key, value is part of the state to be saved to disk
-          //   - srcPath and destPath are lists of files to be copied from src to dest
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (key, value) are the tuples that will be saved to the state.yaml file
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
           def processedState =
             config.functionality.allArguments
               .findAll { it.direction == "output" }
@@ -152,7 +156,7 @@ def publishStatesByConfig(Map args) {
                 // in the state as-is, but is not something that needs 
                 // to be copied from the source path to the dest path
                 if (par.type != "file") {
-                  return [[key: plainName_, value: value, srcPath: [], destPath: []]]
+                  return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
                 }
                 // if the orig state does not contain this filename,
                 // it's an optional argument for which the user specified
@@ -175,15 +179,16 @@ def publishStatesByConfig(Map args) {
                   // the index of the file
                   assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
                   def outputPerFile = value.withIndex().collect{ val, ix ->
-                    def value_ = java.nio.file.Paths.get(filename.replace("*", ix.toString()))
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def value_ = java.nio.file.Paths.get(filename_ix)
                     // if id contains a slash
                     if (yamlDir != null) {
                       value_ = yamlDir.relativize(value_)
                     }
-                    def srcPath = val instanceof File ? val.toPath() : val
-                    [value: value_, srcPath: srcPath, destPath: destPath]
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [value: value_, inputPath: inputPath, outputFilename: filename_ix]
                   }
-                  def transposedOutputs = ["value", "srcPath", "destPath"].collectEntries{ key -> 
+                  def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key -> 
                     [key, outputPerFile.collect{dic -> dic[key]}]
                   }
                   return [[key: plainName_] + transposedOutputs]
@@ -193,22 +198,22 @@ def publishStatesByConfig(Map args) {
                   if (yamlDir != null) {
                     value_ = yamlDir.relativize(value_)
                   }
-                  def srcPath = value instanceof File ? value.toPath() : value
-                  return [[key: plainName_, value: value_, srcPath: [srcPath], destPath: [filename]]]
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
                 }
               }
 
           def updatedState_ = processedState.collectEntries{[it.key, it.value]}
-          def inputFiles_ = processedState.collectMany{it.srcPath}
-          def outputFiles_ = processedState.collectMany{it.destPath}
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
 
           // convert state to yaml blob
           def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
 
-          [id_, yamlBlob_, yamlFilename, inputFiles_, outputFiles_]
+          [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
         }
         | publishStatesProc
     emit: input_ch
   }
   return publishStatesSimpleWf
-}
+}