[WX-965] quote() and squote() engine functions.

centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test

@@ -61,6 +61,21 @@ metadata {
   "outputs.biscayne_new_engine_functions.with_suffixes.1": "bbbS"
   "outputs.biscayne_new_engine_functions.with_suffixes.2": "cccS"
 
+  "outputs.biscayne_new_engine_functions.with_quotes.0": "\"1\""
+  "outputs.biscayne_new_engine_functions.with_quotes.1": "\"2\""
+  "outputs.biscayne_new_engine_functions.with_quotes.2": "\"3\""
+
+  "outputs.biscayne_new_engine_functions.string_with_quotes.0": "\"aaa\""
+  "outputs.biscayne_new_engine_functions.string_with_quotes.1": "\"bbb\""
+  "outputs.biscayne_new_engine_functions.string_with_quotes.2": "\"ccc\""
+
+  "outputs.biscayne_new_engine_functions.with_squotes.0": "'1'"
+  "outputs.biscayne_new_engine_functions.with_squotes.1": "'2'"
+  "outputs.biscayne_new_engine_functions.with_squotes.2": "'3'"
+
+  "outputs.biscayne_new_engine_functions.string_with_squotes.0": "'aaa'"
+  "outputs.biscayne_new_engine_functions.string_with_squotes.1": "'bbb'"
+  "outputs.biscayne_new_engine_functions.string_with_squotes.2": "'ccc'"
   "outputs.biscayne_new_engine_functions.unzipped_a.left.0": "A"
   "outputs.biscayne_new_engine_functions.unzipped_a.right.0": "a"
 

centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl

@@ -4,7 +4,7 @@ workflow biscayne_new_engine_functions {
 
   meta {
     description: "This test makes sure that these functions work in a real workflow"
-    functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key", "suffix", "unzip" ]
+    functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key", "quote", "squote", "sub", "suffix", "unzip" ]
   }
 
   Map[String, Int] x_map_in = {"a": 1, "b": 2, "c": 3}
@@ -17,6 +17,8 @@ workflow biscayne_new_engine_functions {
 
   Array[String] some_strings = ["aaa", "bbb", "ccc"]
 
+  Array[Int] some_ints = [1, 2, 3]
+
   Int smallestInt = 1
   Float smallFloat = 2.718
   Float bigFloat = 3.141
@@ -64,6 +66,16 @@ workflow biscayne_new_engine_functions {
     # =================================================
     Array[String] with_suffixes = suffix("S", some_strings)
 
+    # quote():
+    # =================================================
+    Array[String] with_quotes = quote(some_ints)
+    Array[String] string_with_quotes = quote(some_strings)
+
+    # squote():
+    # =================================================
+    Array[String] with_squotes = squote(some_ints)
+    Array[String] string_with_squotes = squote(some_strings)
+
     # unzip():
     # =================================================
     Pair[Array[String], Array[String]] unzipped_a = unzip(zipped_a)

wdl/model/draft3/src/main/scala/wdl/model/draft3/elements/ExpressionElement.scala

@@ -136,6 +136,8 @@ object ExpressionElement {
   final case class Ceil(param: ExpressionElement) extends OneParamFunctionCallElement
   final case class Round(param: ExpressionElement) extends OneParamFunctionCallElement
   final case class Glob(param: ExpressionElement) extends OneParamFunctionCallElement
+  final case class Quote(param: ExpressionElement) extends OneParamFunctionCallElement
+  final case class SQuote(param: ExpressionElement) extends OneParamFunctionCallElement
   final case class Unzip(param: ExpressionElement) extends OneParamFunctionCallElement
 
   // 1- or 2-param functions:

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/ast2wdlom/AstToNewExpressionElements.scala

@@ -10,7 +10,9 @@ import wdl.model.draft3.elements.ExpressionElement.{
   Keys,
   Max,
   Min,
+  Quote,
   Sep,
+  SQuote,
   SubPosix,
   Suffix,
   Unzip
@@ -28,6 +30,8 @@ object AstToNewExpressionElements {
     "sep" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Sep, "sep"),
     "sub" -> AstNodeToExpressionElement.validateThreeParamEngineFunction(SubPosix, "sub"),
     "suffix" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Suffix, "suffix"),
+    "quote" -> AstNodeToExpressionElement.validateOneParamEngineFunction(Quote, "quote"),
+    "squote" -> AstNodeToExpressionElement.validateOneParamEngineFunction(SQuote, "squote"),
     "unzip" -> AstNodeToExpressionElement.validateOneParamEngineFunction(Unzip, "unzip"),
     "read_object" -> (_ =>
       "read_object is no longer available in this WDL version. Consider using read_json instead".invalidNel

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumers.scala

@@ -76,6 +76,20 @@ object BiscayneExpressionValueConsumers {
         expressionValueConsumer.expressionConsumedValueHooks(a.arg2)(expressionValueConsumer)
   }
 
+  implicit val quoteExpressionValueConsumer: ExpressionValueConsumer[Quote] = new ExpressionValueConsumer[Quote] {
+    override def expressionConsumedValueHooks(a: Quote)(implicit
+      expressionValueConsumer: ExpressionValueConsumer[ExpressionElement]
+    ): Set[UnlinkedConsumedValueHook] =
+      expressionValueConsumer.expressionConsumedValueHooks(a.param)(expressionValueConsumer)
+  }
+
+  implicit val sQuoteExpressionValueConsumer: ExpressionValueConsumer[SQuote] = new ExpressionValueConsumer[SQuote] {
+    override def expressionConsumedValueHooks(a: SQuote)(implicit
+      expressionValueConsumer: ExpressionValueConsumer[ExpressionElement]
+    ): Set[UnlinkedConsumedValueHook] =
+      expressionValueConsumer.expressionConsumedValueHooks(a.param)(expressionValueConsumer)
+  }
+
   implicit val unzipExpressionValueConsumer: ExpressionValueConsumer[Unzip] = new ExpressionValueConsumer[Unzip] {
     override def expressionConsumedValueHooks(a: Unzip)(implicit
       expressionValueConsumer: ExpressionValueConsumer[ExpressionElement]

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/consumed/consumed.scala

@@ -104,6 +104,8 @@ package object consumed {
         case a: AsPairs => a.expressionConsumedValueHooks(expressionValueConsumer)
         case a: CollectByKey => a.expressionConsumedValueHooks(expressionValueConsumer)
         case a: Sep => sepExpressionValueConsumer.expressionConsumedValueHooks(a)(expressionValueConsumer)
+        case a: Quote => a.expressionConsumedValueHooks(expressionValueConsumer)
+        case a: SQuote => a.expressionConsumedValueHooks(expressionValueConsumer)
         case a: Unzip => a.expressionConsumedValueHooks(expressionValueConsumer)
 
         case a: Min => a.expressionConsumedValueHooks(expressionValueConsumer)

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluators.scala

@@ -7,36 +7,36 @@
   Keys,
   Max,
   Min,
+  Quote,
   Sep,
+  SQuote,
   SubPosix,
   Suffix,
   Unzip
 }
 import wdl.model.draft3.graph.expression.FileEvaluator
-import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators
 import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators.{
   threeParameterFunctionPassthroughFileEvaluator,
   twoParameterFunctionPassthroughFileEvaluator
 }
+import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
 
 object BiscayneFileEvaluators {
 
-  implicit val keysFileEvaluator: FileEvaluator[Keys] = EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
-  implicit val asMapFileEvaluator: FileEvaluator[AsMap] =
-    EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
-  implicit val asPairsFileEvaluator: FileEvaluator[AsPairs] =
-    EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
-  implicit val collectByKeyFileEvaluator: FileEvaluator[CollectByKey] =
-    EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
+  implicit val keysFileEvaluator: FileEvaluator[Keys] = singleParameterPassthroughFileEvaluator
+  implicit val asMapFileEvaluator: FileEvaluator[AsMap] = singleParameterPassthroughFileEvaluator
+  implicit val asPairsFileEvaluator: FileEvaluator[AsPairs] = singleParameterPassthroughFileEvaluator
+  implicit val collectByKeyFileEvaluator: FileEvaluator[CollectByKey] = singleParameterPassthroughFileEvaluator
 
   implicit val sepFunctionEvaluator: FileEvaluator[Sep] = twoParameterFunctionPassthroughFileEvaluator[Sep]
   implicit val subPosixFunctionEvaluator: FileEvaluator[SubPosix] =
     threeParameterFunctionPassthroughFileEvaluator[SubPosix]
   implicit val suffixFunctionEvaluator: FileEvaluator[Suffix] = twoParameterFunctionPassthroughFileEvaluator[Suffix]
+  implicit val quoteFunctionEvaluator: FileEvaluator[Quote] = singleParameterPassthroughFileEvaluator
+  implicit val sQuoteFunctionEvaluator: FileEvaluator[SQuote] = singleParameterPassthroughFileEvaluator
 
   implicit val minFunctionEvaluator: FileEvaluator[Min] = twoParameterFunctionPassthroughFileEvaluator[Min]
   implicit val maxFunctionEvaluator: FileEvaluator[Max] = twoParameterFunctionPassthroughFileEvaluator[Max]
 
-  implicit val unzipFunctionEvaluator: FileEvaluator[Unzip] =
-    EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
+  implicit val unzipFunctionEvaluator: FileEvaluator[Unzip] = singleParameterPassthroughFileEvaluator
 }

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/files/files.scala

@@ -146,7 +146,8 @@
         case a: Ceil => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
         case a: Round => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
         case a: Glob => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
-
+        case a: Quote => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
+        case a: SQuote => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
         case a: Size => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
         case a: Basename =>
           a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluators.scala

@@ -121,6 +121,34 @@
       ) mapN { (_, _) => WomArrayType(WomStringType) }
   }
 
+  implicit val quoteFunctionEvaluator: TypeEvaluator[Quote] = new TypeEvaluator[Quote] {
+    override def evaluateType(a: Quote, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit
+      expressionTypeEvaluator: TypeEvaluator[ExpressionElement]
+    ): ErrorOr[WomType] =
+      validateParamType(a.param, linkedValues, WomArrayType(WomAnyType)) flatMap {
+        case WomArrayType(WomNothingType) => WomArrayType(WomNothingType).validNel
+        case WomArrayType(_: WomPrimitiveType) => WomArrayType(WomStringType).validNel
+        case other @ WomArrayType(_) =>
+          s"Cannot invoke quote on type Array[${other.stableName}]. Expected an Array[Primitive type]".invalidNel
+        case other =>
+          s"Cannot invoke quote on type ${other.stableName}. Expected an Array[Primitive type]".invalidNel
+      }
+  }
+
+  implicit val sQuoteFunctionEvaluator: TypeEvaluator[SQuote] = new TypeEvaluator[SQuote] {
+    override def evaluateType(a: SQuote, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit
+      expressionTypeEvaluator: TypeEvaluator[ExpressionElement]
+    ): ErrorOr[WomType] =
+      validateParamType(a.param, linkedValues, WomArrayType(WomAnyType)) flatMap {
+        case WomArrayType(WomNothingType) => WomArrayType(WomNothingType).validNel
+        case WomArrayType(_: WomPrimitiveType) => WomArrayType(WomStringType).validNel
+        case other @ WomArrayType(_) =>
+          s"Cannot invoke squote on type Array[${other.stableName}]. Expected an Array[Primitive type]".invalidNel
+        case other =>
+          s"Cannot invoke squote on type ${other.stableName}. Expected an Array[Primitive type]".invalidNel
+      }
+  }
+
   implicit val unzipFunctionEvaluator: TypeEvaluator[Unzip] = new TypeEvaluator[Unzip] {
     override def evaluateType(a: Unzip, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit
       expressionTypeEvaluator: TypeEvaluator[ExpressionElement]

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/types/types.scala

@@ -91,7 +91,8 @@ package object types {
         case a: Ceil => a.evaluateType(linkedValues)(typeEvaluator)
         case a: Round => a.evaluateType(linkedValues)(typeEvaluator)
         case a: Glob => a.evaluateType(linkedValues)(typeEvaluator)
-
+        case a: Quote => a.evaluateType(linkedValues)(typeEvaluator)
+        case a: SQuote => a.evaluateType(linkedValues)(typeEvaluator)
         case a: Size => a.evaluateType(linkedValues)(typeEvaluator)
         case a: Basename => a.evaluateType(linkedValues)(typeEvaluator)
 

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/BiscayneValueEvaluators.scala

@@ -269,6 +269,54 @@ object BiscayneValueEvaluators {
       }
   }
 
+  /**
+   * Quote: Given an array of primitives, produce a new array in which all elements of the original are in quotes (").
+   * https://github.com/openwdl/wdl/blob/main/versions/1.1/SPEC.md#-arraystring-quotearrayp
+   * input: Array[Primitive]
+   * output: Array[String]
+   */
+  implicit val quoteFunctionEvaluator: ValueEvaluator[Quote] = new ValueEvaluator[Quote] {
+    override def evaluateValue(a: Quote,
+                               inputs: Map[String, WomValue],
+                               ioFunctionSet: IoFunctionSet,
+                               forCommandInstantiationOptions: Option[ForCommandInstantiationOptions]
+    )(implicit expressionValueEvaluator: ValueEvaluator[ExpressionElement]): ErrorOr[EvaluatedValue[WomArray]] =
+      processValidatedSingleValue[WomArray, WomArray](
+        expressionValueEvaluator.evaluateValue(a.param, inputs, ioFunctionSet, forCommandInstantiationOptions)(
+          expressionValueEvaluator
+        )
+      ) { arr =>
+        EvaluatedValue(
+          WomArray(arr.value.map(v => WomString("\"" + v.valueString.replaceAll(""""""", "\"") + "\""))),
+          Seq.empty
+        ).validNel
+      }
+  }
+
+  /**
+   * SQuote: Given an array of primitives, produce a new array in which all elements of the original are in single quotes (').
+   * https://github.com/openwdl/wdl/blob/main/versions/1.1/SPEC.md#-arraystring-squotearrayp
+   * input: Array[Primitive]
+   * output: Array[String]
+   */
+  implicit val sQuoteFunctionEvaluator: ValueEvaluator[SQuote] = new ValueEvaluator[SQuote] {
+    override def evaluateValue(a: SQuote,
+                               inputs: Map[String, WomValue],
+                               ioFunctionSet: IoFunctionSet,
+                               forCommandInstantiationOptions: Option[ForCommandInstantiationOptions]
+    )(implicit expressionValueEvaluator: ValueEvaluator[ExpressionElement]): ErrorOr[EvaluatedValue[WomArray]] =
+      processValidatedSingleValue[WomArray, WomArray](
+        expressionValueEvaluator.evaluateValue(a.param, inputs, ioFunctionSet, forCommandInstantiationOptions)(
+          expressionValueEvaluator
+        )
+      ) { arr =>
+        EvaluatedValue(
+          WomArray(arr.value.map(v => WomString("\'" + v.valueString.replaceAll("""'""", "\'") + "\'"))),
+          Seq.empty
+        ).validNel
+      }
+  }
+
   /**
    * Unzip: Creates a pair of arrays, the first containing the elements from the left members of an array of pairs,
    * and the second containing the right members. This is the inverse of the zip function.

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/linking/expression/values/values.scala

@@ -156,6 +156,10 @@ package object values {
         case a: Round =>
           a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
         case a: Glob => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
+        case a: Quote =>
+          a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
+        case a: SQuote =>
+          a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
 
         case a: Size => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
         case a: Basename =>

wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/Ast2WdlomSpec.scala

@@ -109,6 +109,18 @@ class Ast2WdlomSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
     expr shouldBeValid (Suffix(IdentifierLookup("some_str"), IdentifierLookup("some_arr")))
   }
 
+  it should "parse the new quote function" in {
+    val str = "quote(some_arr)"
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+    expr shouldBeValid (Quote(IdentifierLookup("some_arr")))
+  }
+
+  it should "parse the new squote function" in {
+    val str = "squote(some_arr)"
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+    expr shouldBeValid (SQuote(IdentifierLookup("some_arr")))
+  }
+
   it should "parse the new unzip function" in {
     val str = "unzip(some_array_of_pairs)"
     val expr = fromString[ExpressionElement](str, parser.parse_e)

wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/consumed/BiscayneExpressionValueConsumersSpec.scala

@@ -85,6 +85,24 @@ class BiscayneExpressionValueConsumersSpec extends AnyFlatSpec with CromwellTime
     }
   }
 
+  it should "discover an array variable lookup within a quote() call" in {
+    val str = """ quote(my_array) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_array")))
+    }
+  }
+
+  it should "discover an array variable lookup within a squote() call" in {
+    val str = """ squote(my_array) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_array")))
+    }
+  }
+
   it should "discover an array variable lookup within a unzip() call" in {
     val str = """ unzip(my_array_of_pairs) """
     val expr = fromString[ExpressionElement](str, parser.parse_e)

wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/files/BiscayneFileEvaluatorSpec.scala

@@ -68,6 +68,28 @@ class BiscayneFileEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit
     }
   }
 
+  it should "discover the file which would be required to evaluate a quote() function" in {
+    val str = """ quote(read_lines("foo.txt")) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.predictFilesNeededToEvaluate(Map.empty, NoIoFunctionSet, WomStringType) shouldBeValid Set(
+        WomSingleFile("foo.txt")
+      )
+    }
+  }
+
+  it should "discover the file which would be required to evaluate a squote() function" in {
+    val str = """ squote(read_lines("foo.txt")) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.predictFilesNeededToEvaluate(Map.empty, NoIoFunctionSet, WomStringType) shouldBeValid Set(
+        WomSingleFile("foo.txt")
+      )
+    }
+  }
+
   it should "discover the file which would be required to evaluate a unzip() function" in {
     val str = """ unzip(read_lines("foo.txt")) """
     val expr = fromString[ExpressionElement](str, parser.parse_e)

wdl/transforms/biscayne/src/test/scala/wdl/transforms/biscayne/linking/expression/types/BiscayneTypeEvaluatorSpec.scala

@@ -74,6 +74,42 @@ class BiscayneTypeEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit
     }
   }
 
+  it should "evaluate the type of a quote() function as Array[String]" in {
+    val str = """ quote([1, 2, 3]) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomStringType)
+    }
+  }
+
+  it should "evaluate the type of a quote() function with an empty array as Array[Nothing]" in {
+    val str = """ quote([]) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomNothingType)
+    }
+  }
+
+  it should "evaluate the type of a squote() function as Array[String]" in {
+    val str = """ squote([1, 2, 3]) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomStringType)
+    }
+  }
+
+  it should "evaluate the type of an squote() function with an empty array as Array[Nothing]" in {
+    val str = """ squote([]) """
+    val expr = fromString[ExpressionElement](str, parser.parse_e)
+
+    expr.shouldBeValidPF { case e =>
+      e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomNothingType)
+    }
+  }
+
   it should "evaluate the type of an unzip() function as Pair[Array[X], Array[Y]]" in {
     val string_and_int = """ unzip([("one", 1),("two", 2),("three", 3)]) """
     val string_and_int_expr = fromString[ExpressionElement](string_and_int, parser.parse_e)