broadinstitute · aednichols · Jul 31, 2023 · May 3, 2023 · May 3, 2023 · May 4, 2023
@@ -24,7 +24,11 @@ jobs:
       #Each will be launched on its own runner so they can occur in parallel.
       #Friendly names are displayed on the Github UI and aren't used anywhere else.
       matrix:
+        # Batch test fixes to land later
         include:
+          # - build_type: centaurGcpBatch
+          #   build_mysql: 5.7
+          #   friendly_name: Centaur GCP Batch with MySQL 5.7
           - build_type: centaurPapiV2beta
             build_mysql: 5.7
             friendly_name: Centaur Papi V2 Beta with MySQL 5.7

@@ -233,6 +233,19 @@ lazy val googlePipelinesV2Beta = (project in backendRoot / "google" / "pipelines
   .dependsOn(core % "test->test")
   .dependsOn(common % "test->test")
 
+lazy val googleBatch = (project in backendRoot / "google" / "batch")
+  .withLibrarySettings("cromwell-google-batch-backend")
+  .dependsOn(backend)
+  .dependsOn(gcsFileSystem)
+  .dependsOn(drsFileSystem)
+  .dependsOn(sraFileSystem)
+  .dependsOn(httpFileSystem)
+  .dependsOn(backend % "test->test")
+  .dependsOn(gcsFileSystem % "test->test")
+  .dependsOn(services % "test->test")
+  .dependsOn(common % "test->test")
+  .dependsOn(core % "test->test")
+
 lazy val awsBackend = (project in backendRoot / "aws")
   .withLibrarySettings("cromwell-aws-backend")
   .dependsOn(backend)
@@ -392,6 +405,7 @@ lazy val server = project
   .dependsOn(engine)
   .dependsOn(googlePipelinesV2Alpha1)
   .dependsOn(googlePipelinesV2Beta)
+  .dependsOn(googleBatch)
   .dependsOn(awsBackend)
   .dependsOn(tesBackend)
   .dependsOn(cromwellApiClient)
@@ -431,6 +445,7 @@ lazy val root = (project in file("."))
   .aggregate(googlePipelinesCommon)
   .aggregate(googlePipelinesV2Alpha1)
   .aggregate(googlePipelinesV2Beta)
+  .aggregate(googleBatch)
   .aggregate(httpFileSystem)
   .aggregate(languageFactoryCore)
   .aggregate(perf)

@@ -2,6 +2,7 @@ name: draft3_read_file_limits
 testFormat: workflowfailure
 workflowType: WDL
 workflowTypeVersion: 1.0
+tags: [batchexclude]
 
 files {
   workflow: wdl_draft3/read_file_limits/read_file_limits.wdl

@@ -9,6 +9,7 @@
 
 name: long_cmd
 testFormat: workflowsuccess
+tags: [batchexclude]
 
 files {
   workflow: long_cmd/long_cmd.wdl

@@ -1,5 +1,6 @@
 name: read_file_limits
 testFormat: workflowfailure
+tags: [batchexclude]
 
 files {
   workflow: read_file_limits/read_file_limits.wdl

@@ -1,5 +1,6 @@
 name: relative_output_paths_colliding
 testFormat: workflowfailure
+tags: [batchexclude]
 
 files {
   workflow: relative_output_paths_colliding/workflow_output_paths_colliding.wdl

@@ -1,5 +1,6 @@
 name: standard_output_paths_colliding_prevented
 testFormat: workflowsuccess
+tags: [batchexclude]
 
 files {
   workflow: standard_output_paths_colliding_prevented/workflow_output_paths_colliding.wdl

@@ -0,0 +1,104 @@
+# This is an example of how you can use the Google Cloud Batch backend
+# provider. *This is not a complete configuration file!* The
+# content here should be copy pasted into the backend -> providers section
+# of cromwell.example.backends/cromwell.examples.conf in the root of the repository.
+# You should uncomment lines that you want to define, and read carefully to customize
+# the file.
+
+# Documentation
+# https://cromwell.readthedocs.io/en/stable/backends/Google/
+
+backend {
+  default = GCPBATCH
+
+  providers {
+    GCPBATCH {
+      actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
+      config {
+        # Google project
+        project = "my-cromwell-workflows"
+
+        # Base bucket for workflow executions
+        root = "gs://my-cromwell-workflows-bucket"
+
+        # Polling for completion backs-off gradually for slower-running jobs.
+        # This is the maximum polling interval (in seconds):
+        maximum-polling-interval = 600
+
+        # Optional Dockerhub Credentials. Can be used to access private docker images.
+        dockerhub {
+          # account = ""
+          # token = ""
+        }
+
+        # Optional configuration to use high security network (Virtual Private Cloud) for running jobs.
+        # See https://cromwell.readthedocs.io/en/stable/backends/Google/ for more details.
+        # virtual-private-cloud {
+        #  network-label-key = "network-key"
+        #  auth = "application-default"
+        # }
+
+        # Global pipeline timeout
+        # Defaults to 7 days; max 30 days
+        # batch-timeout = 7 days
+
+        genomics {
+          # A reference to an auth defined in the `google` stanza at the top.  This auth is used to create
+          # Batch Jobs and manipulate auth JSONs.
+          auth = "application-default"
+
+
+          // alternative service account to use on the launched compute instance
+          // NOTE: If combined with service account authorization, both that service account and this service account
+          // must be able to read and write to the 'root' GCS path
+          compute-service-account = "default"
+
+          # Location to submit jobs to Batch and store job metadata.
+          location = "us-central1"
+
+          # Specifies the minimum file size for `gsutil cp` to use parallel composite uploads during delocalization.
+          # Parallel composite uploads can result in a significant improvement in delocalization speed for large files
+          # but may introduce complexities in downloading such files from GCS, please see
+          # https://cloud.google.com/storage/docs/gsutil/commands/cp#parallel-composite-uploads for more information.
+          #
+          # If set to 0 parallel composite uploads are turned off. The default Cromwell configuration turns off
+          # parallel composite uploads, this sample configuration turns it on for files of 150M or larger.
+          parallel-composite-upload-threshold="150M"
+        }
+
+        filesystems {
+          gcs {
+            # A reference to a potentially different auth for manipulating files via engine functions.
+            auth = "application-default"
+            # Google project which will be billed for the requests
+            project = "google-billing-project"
+
+            caching {
+              # When a cache hit is found, the following duplication strategy will be followed to use the cached outputs
+              # Possible values: "copy", "reference". Defaults to "copy"
+              # "copy": Copy the output files
+              # "reference": DO NOT copy the output files but point to the original output files instead.
+              #              Will still make sure than all the original output files exist and are accessible before
+              #              going forward with the cache hit.
+              duplication-strategy = "copy"
+            }
+          }
+        }
+
+        default-runtime-attributes {
+          cpu: 1
+          failOnStderr: false
+          continueOnReturnCode: 0
+          memory: "2048 MB"
+          bootDiskSizeGb: 10
+          # Allowed to be a String, or a list of Strings
+          disks: "local-disk 10 SSD"
+          noAddress: false
+          preemptible: 0
+          zones: ["us-central1-a", "us-central1-b"]
+        }
+
+      }
+    }
+  }
+}