Hybrid Batch + Smart Downloads

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 ARG DEBIAN_FRONTEND=noninteractive
 # currently, there is an issue in v1.5.3 so we can't upgrade until it is resolved https://github.com/chanzuckerberg/miniwdl/issues/607
 ARG MINIWDL_VERSION=1.5.2
@@ -61,7 +61,6 @@ ADD miniwdl-plugins miniwdl-plugins
 
 RUN pip install miniwdl-plugins/s3upload
 RUN pip install miniwdl-plugins/sfn_wdl
-RUN pip install miniwdl-plugins/s3parcp_download
 
 RUN cd /usr/bin; curl -O https://amazon-ecr-credential-helper-releases.s3.amazonaws.com/0.4.0/linux-amd64/docker-credential-ecr-login
 RUN chmod +x /usr/bin/docker-credential-ecr-login

foo.wdl

@@ -0,0 +1,49 @@
+version 1.0
+
+task say_hello {
+  input {
+    String salutation
+    String name
+    String docker_image_id
+  }
+  command <<<
+    echo "~{salutation} ~{name}" > "~{name}.txt"
+  >>>
+  output {
+    String greeting = "~{name}.txt"
+  }
+  runtime {
+    maxRetries: 3
+    docker: docker_image_id
+  }
+}
+
+# task merge {
+#   input {
+#     Array[String] greetings
+#   }
+#   command <<<
+#     echo "~{sep=', ' greetings}"
+#   >>>
+#   output {
+#     String combined = stdout()
+#   }
+# }
+
+workflow scatter_example {
+  input {
+    Array[String] name_array = ["Joe", "Bob", "Fred"]
+    String salutation = "hello"
+    String docker_image_id = "ubuntu:20.04"
+  }
+
+  scatter (name in name_array) {
+    call say_hello { input: name = name, salutation = salutation, docker_image_id = docker_image_id }
+  }
+
+  # call merge { input: greetings = say_hello.greeting }
+
+  output {
+    Array[String] combined = say_hello.greeting
+  }
+}

main.tf

@@ -49,7 +49,7 @@ locals {
 module "sfn" {
   source                        = "./terraform/modules/swipe-sfn"
   app_name                      = var.app_name
-  batch_job_docker_image        = "ghcr.io/chanzuckerberg/swipe:${chomp(local.version)}"
+  batch_job_docker_image        = "732052188396.dkr.ecr.us-west-2.amazonaws.com/swipe:9ff3d52f958d82eb7c1c1cc37ceee321031a3a94"
   batch_spot_job_queue_arn      = module.batch_queue.batch_spot_job_queue_arn
   batch_on_demand_job_queue_arn = module.batch_queue.batch_on_demand_job_queue_arn
   miniwdl_dir                   = var.miniwdl_dir