Skip to content

Commit

Permalink
[WX-965] quote() and squote() engine functions. (#7375)
Browse files Browse the repository at this point in the history
Co-authored-by: Ryan Saperstein <rsaperst@wmd55-b47.hsd1.ma.comcast.net>
Co-authored-by: Janet Gainer-Dewar <jdewar@broadinstitute.org>
  • Loading branch information
3 people authored Feb 29, 2024
1 parent e1d5ff4 commit db0dd2d
Show file tree
Hide file tree
Showing 33 changed files with 788 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,21 @@ metadata {
"outputs.biscayne_new_engine_functions.with_suffixes.1": "bbbS"
"outputs.biscayne_new_engine_functions.with_suffixes.2": "cccS"

"outputs.biscayne_new_engine_functions.with_quotes.0": "\"1\""
"outputs.biscayne_new_engine_functions.with_quotes.1": "\"2\""
"outputs.biscayne_new_engine_functions.with_quotes.2": "\"3\""

"outputs.biscayne_new_engine_functions.string_with_quotes.0": "\"aaa\""
"outputs.biscayne_new_engine_functions.string_with_quotes.1": "\"bbb\""
"outputs.biscayne_new_engine_functions.string_with_quotes.2": "\"ccc\""

"outputs.biscayne_new_engine_functions.with_squotes.0": "'1'"
"outputs.biscayne_new_engine_functions.with_squotes.1": "'2'"
"outputs.biscayne_new_engine_functions.with_squotes.2": "'3'"

"outputs.biscayne_new_engine_functions.string_with_squotes.0": "'aaa'"
"outputs.biscayne_new_engine_functions.string_with_squotes.1": "'bbb'"
"outputs.biscayne_new_engine_functions.string_with_squotes.2": "'ccc'"
"outputs.biscayne_new_engine_functions.unzipped_a.left.0": "A"
"outputs.biscayne_new_engine_functions.unzipped_a.right.0": "a"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ workflow biscayne_new_engine_functions {

meta {
description: "This test makes sure that these functions work in a real workflow"
functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key", "suffix", "unzip" ]
functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key", "quote", "squote", "sub", "suffix", "unzip" ]
}

Map[String, Int] x_map_in = {"a": 1, "b": 2, "c": 3}
Expand All @@ -17,6 +17,8 @@ workflow biscayne_new_engine_functions {

Array[String] some_strings = ["aaa", "bbb", "ccc"]

Array[Int] some_ints = [1, 2, 3]

Int smallestInt = 1
Float smallFloat = 2.718
Float bigFloat = 3.141
Expand Down Expand Up @@ -64,6 +66,16 @@ workflow biscayne_new_engine_functions {
# =================================================
Array[String] with_suffixes = suffix("S", some_strings)

# quote():
# =================================================
Array[String] with_quotes = quote(some_ints)
Array[String] string_with_quotes = quote(some_strings)

# squote():
# =================================================
Array[String] with_squotes = squote(some_ints)
Array[String] string_with_squotes = squote(some_strings)

# unzip():
# =================================================
Pair[Array[String], Array[String]] unzipped_a = unzip(zipped_a)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,8 @@ object ExpressionElement {
final case class Ceil(param: ExpressionElement) extends OneParamFunctionCallElement
final case class Round(param: ExpressionElement) extends OneParamFunctionCallElement
final case class Glob(param: ExpressionElement) extends OneParamFunctionCallElement
final case class Quote(param: ExpressionElement) extends OneParamFunctionCallElement
final case class SQuote(param: ExpressionElement) extends OneParamFunctionCallElement
final case class Unzip(param: ExpressionElement) extends OneParamFunctionCallElement

// 1- or 2-param functions:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import wdl.model.draft3.elements.ExpressionElement.{
Keys,
Max,
Min,
Quote,
Sep,
SQuote,
SubPosix,
Suffix,
Unzip
Expand All @@ -28,6 +30,8 @@ object AstToNewExpressionElements {
"sep" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Sep, "sep"),
"sub" -> AstNodeToExpressionElement.validateThreeParamEngineFunction(SubPosix, "sub"),
"suffix" -> AstNodeToExpressionElement.validateTwoParamEngineFunction(Suffix, "suffix"),
"quote" -> AstNodeToExpressionElement.validateOneParamEngineFunction(Quote, "quote"),
"squote" -> AstNodeToExpressionElement.validateOneParamEngineFunction(SQuote, "squote"),
"unzip" -> AstNodeToExpressionElement.validateOneParamEngineFunction(Unzip, "unzip"),
"read_object" -> (_ =>
"read_object is no longer available in this WDL version. Consider using read_json instead".invalidNel
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,20 @@ object BiscayneExpressionValueConsumers {
expressionValueConsumer.expressionConsumedValueHooks(a.arg2)(expressionValueConsumer)
}

implicit val quoteExpressionValueConsumer: ExpressionValueConsumer[Quote] = new ExpressionValueConsumer[Quote] {
override def expressionConsumedValueHooks(a: Quote)(implicit
expressionValueConsumer: ExpressionValueConsumer[ExpressionElement]
): Set[UnlinkedConsumedValueHook] =
expressionValueConsumer.expressionConsumedValueHooks(a.param)(expressionValueConsumer)
}

implicit val sQuoteExpressionValueConsumer: ExpressionValueConsumer[SQuote] = new ExpressionValueConsumer[SQuote] {
override def expressionConsumedValueHooks(a: SQuote)(implicit
expressionValueConsumer: ExpressionValueConsumer[ExpressionElement]
): Set[UnlinkedConsumedValueHook] =
expressionValueConsumer.expressionConsumedValueHooks(a.param)(expressionValueConsumer)
}

implicit val unzipExpressionValueConsumer: ExpressionValueConsumer[Unzip] = new ExpressionValueConsumer[Unzip] {
override def expressionConsumedValueHooks(a: Unzip)(implicit
expressionValueConsumer: ExpressionValueConsumer[ExpressionElement]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ package object consumed {
case a: AsPairs => a.expressionConsumedValueHooks(expressionValueConsumer)
case a: CollectByKey => a.expressionConsumedValueHooks(expressionValueConsumer)
case a: Sep => sepExpressionValueConsumer.expressionConsumedValueHooks(a)(expressionValueConsumer)
case a: Quote => a.expressionConsumedValueHooks(expressionValueConsumer)
case a: SQuote => a.expressionConsumedValueHooks(expressionValueConsumer)
case a: Unzip => a.expressionConsumedValueHooks(expressionValueConsumer)

case a: Min => a.expressionConsumedValueHooks(expressionValueConsumer)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,36 @@ import wdl.model.draft3.elements.ExpressionElement.{
Keys,
Max,
Min,
Quote,
Sep,
SQuote,
SubPosix,
Suffix,
Unzip
}
import wdl.model.draft3.graph.expression.FileEvaluator
import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators
import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators.{
threeParameterFunctionPassthroughFileEvaluator,
twoParameterFunctionPassthroughFileEvaluator
}
import wdl.transforms.base.linking.expression.files.EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator

object BiscayneFileEvaluators {

implicit val keysFileEvaluator: FileEvaluator[Keys] = EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
implicit val asMapFileEvaluator: FileEvaluator[AsMap] =
EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
implicit val asPairsFileEvaluator: FileEvaluator[AsPairs] =
EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
implicit val collectByKeyFileEvaluator: FileEvaluator[CollectByKey] =
EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
implicit val keysFileEvaluator: FileEvaluator[Keys] = singleParameterPassthroughFileEvaluator
implicit val asMapFileEvaluator: FileEvaluator[AsMap] = singleParameterPassthroughFileEvaluator
implicit val asPairsFileEvaluator: FileEvaluator[AsPairs] = singleParameterPassthroughFileEvaluator
implicit val collectByKeyFileEvaluator: FileEvaluator[CollectByKey] = singleParameterPassthroughFileEvaluator

implicit val sepFunctionEvaluator: FileEvaluator[Sep] = twoParameterFunctionPassthroughFileEvaluator[Sep]
implicit val subPosixFunctionEvaluator: FileEvaluator[SubPosix] =
threeParameterFunctionPassthroughFileEvaluator[SubPosix]
implicit val suffixFunctionEvaluator: FileEvaluator[Suffix] = twoParameterFunctionPassthroughFileEvaluator[Suffix]
implicit val quoteFunctionEvaluator: FileEvaluator[Quote] = singleParameterPassthroughFileEvaluator
implicit val sQuoteFunctionEvaluator: FileEvaluator[SQuote] = singleParameterPassthroughFileEvaluator

implicit val minFunctionEvaluator: FileEvaluator[Min] = twoParameterFunctionPassthroughFileEvaluator[Min]
implicit val maxFunctionEvaluator: FileEvaluator[Max] = twoParameterFunctionPassthroughFileEvaluator[Max]

implicit val unzipFunctionEvaluator: FileEvaluator[Unzip] =
EngineFunctionEvaluators.singleParameterPassthroughFileEvaluator
implicit val unzipFunctionEvaluator: FileEvaluator[Unzip] = singleParameterPassthroughFileEvaluator
}
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,8 @@ package object files {
case a: Ceil => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
case a: Round => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
case a: Glob => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)

case a: Quote => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
case a: SQuote => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
case a: Size => a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
case a: Basename =>
a.predictFilesNeededToEvaluate(inputs, ioFunctionSet, coerceTo)(fileEvaluator, valueEvaluator)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,34 @@ object BiscayneTypeEvaluators {
) mapN { (_, _) => WomArrayType(WomStringType) }
}

implicit val quoteFunctionEvaluator: TypeEvaluator[Quote] = new TypeEvaluator[Quote] {
override def evaluateType(a: Quote, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit
expressionTypeEvaluator: TypeEvaluator[ExpressionElement]
): ErrorOr[WomType] =
validateParamType(a.param, linkedValues, WomArrayType(WomAnyType)) flatMap {
case WomArrayType(WomNothingType) => WomArrayType(WomNothingType).validNel
case WomArrayType(_: WomPrimitiveType) => WomArrayType(WomStringType).validNel
case other @ WomArrayType(_) =>
s"Cannot invoke quote on type Array[${other.stableName}]. Expected an Array of primitive type".invalidNel
case other =>
s"Cannot invoke quote on type ${other.stableName}. Expected an Array of primitive type".invalidNel
}
}

implicit val sQuoteFunctionEvaluator: TypeEvaluator[SQuote] = new TypeEvaluator[SQuote] {
override def evaluateType(a: SQuote, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit
expressionTypeEvaluator: TypeEvaluator[ExpressionElement]
): ErrorOr[WomType] =
validateParamType(a.param, linkedValues, WomArrayType(WomAnyType)) flatMap {
case WomArrayType(WomNothingType) => WomArrayType(WomNothingType).validNel
case WomArrayType(_: WomPrimitiveType) => WomArrayType(WomStringType).validNel
case other @ WomArrayType(_) =>
s"Cannot invoke squote on type Array[${other.stableName}]. Expected an Array of primitive type".invalidNel
case other =>
s"Cannot invoke squote on type ${other.stableName}. Expected an Array of primitive type".invalidNel
}
}

implicit val unzipFunctionEvaluator: TypeEvaluator[Unzip] = new TypeEvaluator[Unzip] {
override def evaluateType(a: Unzip, linkedValues: Map[UnlinkedConsumedValueHook, GeneratedValueHandle])(implicit
expressionTypeEvaluator: TypeEvaluator[ExpressionElement]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ package object types {
case a: Ceil => a.evaluateType(linkedValues)(typeEvaluator)
case a: Round => a.evaluateType(linkedValues)(typeEvaluator)
case a: Glob => a.evaluateType(linkedValues)(typeEvaluator)

case a: Quote => a.evaluateType(linkedValues)(typeEvaluator)
case a: SQuote => a.evaluateType(linkedValues)(typeEvaluator)
case a: Size => a.evaluateType(linkedValues)(typeEvaluator)
case a: Basename => a.evaluateType(linkedValues)(typeEvaluator)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,54 @@ object BiscayneValueEvaluators {
}
}

/**
* Quote: Given an array of primitives, produce a new array in which all elements of the original are in quotes (").
* https://github.com/openwdl/wdl/blob/main/versions/1.1/SPEC.md#-arraystring-quotearrayp
* input: Array[Primitive]
* output: Array[String]
*/
implicit val quoteFunctionEvaluator: ValueEvaluator[Quote] = new ValueEvaluator[Quote] {
override def evaluateValue(a: Quote,
inputs: Map[String, WomValue],
ioFunctionSet: IoFunctionSet,
forCommandInstantiationOptions: Option[ForCommandInstantiationOptions]
)(implicit expressionValueEvaluator: ValueEvaluator[ExpressionElement]): ErrorOr[EvaluatedValue[WomArray]] =
processValidatedSingleValue[WomArray, WomArray](
expressionValueEvaluator.evaluateValue(a.param, inputs, ioFunctionSet, forCommandInstantiationOptions)(
expressionValueEvaluator
)
) { arr =>
EvaluatedValue(
WomArray(arr.value.map(v => WomString("\"" + v.valueString.replaceAll(""""""", "\"") + "\""))),
Seq.empty
).validNel
}
}

/**
* SQuote: Given an array of primitives, produce a new array in which all elements of the original are in single quotes (').
* https://github.com/openwdl/wdl/blob/main/versions/1.1/SPEC.md#-arraystring-squotearrayp
* input: Array[Primitive]
* output: Array[String]
*/
implicit val sQuoteFunctionEvaluator: ValueEvaluator[SQuote] = new ValueEvaluator[SQuote] {
override def evaluateValue(a: SQuote,
inputs: Map[String, WomValue],
ioFunctionSet: IoFunctionSet,
forCommandInstantiationOptions: Option[ForCommandInstantiationOptions]
)(implicit expressionValueEvaluator: ValueEvaluator[ExpressionElement]): ErrorOr[EvaluatedValue[WomArray]] =
processValidatedSingleValue[WomArray, WomArray](
expressionValueEvaluator.evaluateValue(a.param, inputs, ioFunctionSet, forCommandInstantiationOptions)(
expressionValueEvaluator
)
) { arr =>
EvaluatedValue(
WomArray(arr.value.map(v => WomString("\'" + v.valueString.replaceAll("""'""", "\'") + "\'"))),
Seq.empty
).validNel
}
}

/**
* Unzip: Creates a pair of arrays, the first containing the elements from the left members of an array of pairs,
* and the second containing the right members. This is the inverse of the zip function.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ package object values {
case a: Round =>
a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
case a: Glob => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
case a: Quote =>
a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
case a: SQuote =>
a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)

case a: Size => a.evaluateValue(inputs, ioFunctionSet, forCommandInstantiationOptions)(expressionValueEvaluator)
case a: Basename =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,18 @@ class Ast2WdlomSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers {
expr shouldBeValid (Suffix(IdentifierLookup("some_str"), IdentifierLookup("some_arr")))
}

it should "parse the new quote function" in {
val str = "quote(some_arr)"
val expr = fromString[ExpressionElement](str, parser.parse_e)
expr shouldBeValid (Quote(IdentifierLookup("some_arr")))
}

it should "parse the new squote function" in {
val str = "squote(some_arr)"
val expr = fromString[ExpressionElement](str, parser.parse_e)
expr shouldBeValid (SQuote(IdentifierLookup("some_arr")))
}

it should "parse the new unzip function" in {
val str = "unzip(some_array_of_pairs)"
val expr = fromString[ExpressionElement](str, parser.parse_e)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,24 @@ class BiscayneExpressionValueConsumersSpec extends AnyFlatSpec with CromwellTime
}
}

it should "discover an array variable lookup within a quote() call" in {
val str = """ quote(my_array) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_array")))
}
}

it should "discover an array variable lookup within a squote() call" in {
val str = """ squote(my_array) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.expressionConsumedValueHooks should be(Set(UnlinkedIdentifierHook("my_array")))
}
}

it should "discover an array variable lookup within a unzip() call" in {
val str = """ unzip(my_array_of_pairs) """
val expr = fromString[ExpressionElement](str, parser.parse_e)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,28 @@ class BiscayneFileEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit
}
}

it should "discover the file which would be required to evaluate a quote() function" in {
val str = """ quote(read_lines("foo.txt")) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.predictFilesNeededToEvaluate(Map.empty, NoIoFunctionSet, WomStringType) shouldBeValid Set(
WomSingleFile("foo.txt")
)
}
}

it should "discover the file which would be required to evaluate a squote() function" in {
val str = """ squote(read_lines("foo.txt")) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.predictFilesNeededToEvaluate(Map.empty, NoIoFunctionSet, WomStringType) shouldBeValid Set(
WomSingleFile("foo.txt")
)
}
}

it should "discover the file which would be required to evaluate a unzip() function" in {
val str = """ unzip(read_lines("foo.txt")) """
val expr = fromString[ExpressionElement](str, parser.parse_e)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,42 @@ class BiscayneTypeEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec wit
}
}

it should "evaluate the type of a quote() function as Array[String]" in {
val str = """ quote([1, 2, 3]) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomStringType)
}
}

it should "evaluate the type of a quote() function with an empty array as Array[Nothing]" in {
val str = """ quote([]) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomNothingType)
}
}

it should "evaluate the type of a squote() function as Array[String]" in {
val str = """ squote([1, 2, 3]) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomStringType)
}
}

it should "evaluate the type of an squote() function with an empty array as Array[Nothing]" in {
val str = """ squote([]) """
val expr = fromString[ExpressionElement](str, parser.parse_e)

expr.shouldBeValidPF { case e =>
e.evaluateType(Map.empty) shouldBeValid WomArrayType(WomNothingType)
}
}

it should "evaluate the type of an unzip() function as Pair[Array[X], Array[Y]]" in {
val string_and_int = """ unzip([("one", 1),("two", 2),("three", 3)]) """
val string_and_int_expr = fromString[ExpressionElement](string_and_int, parser.parse_e)
Expand Down
Loading

0 comments on commit db0dd2d

Please sign in to comment.