Modified cromwell for aws backend

.circleci/config.yml

@@ -19,10 +19,10 @@ update_apt: &update_apt
     sudo apt-get update
 
 
-install_python3: &install_python3_java
-  name: Install python3, pip3, java
+install_python3: &install_python3
+  name: Install python3, pip3
   command: |
-    sudo apt-get install software-properties-common git wget curl default-jre -y
+    sudo apt-get install software-properties-common git wget curl -y
     sudo add-apt-repository ppa:deadsnakes/ppa -y
     sudo apt-get update && sudo apt-get install python3.6 -y
     sudo wget --no-check-certificate https://bootstrap.pypa.io/get-pip.py
@@ -61,17 +61,26 @@ install_aws_lib: &install_aws_lib
     sudo pip3 install boto3 awscli
 
 
+install_java: &install_java
+  name: Install openjdk-11
+  command: |
+    sudo add-apt-repository ppa:openjdk-r/ppa -y
+    sudo apt-get update && sudo apt-get install openjdk-11-jdk -y
+    # automatically set 11 as default java
+    sudo update-java-alternatives -a
+
 jobs:
   pytest:
     <<: *machine_defaults
     steps:
       - checkout
       - run: *update_apt
-      - run: *install_python3_java
+      - run: *install_python3
       - run: *install_singularity
       - run: *install_py3_packages
       - run: *install_gcs_lib
       - run: *install_aws_lib
+      - run: *install_java
       - run:
           no_output_timeout: 60m
           command: |

README.md

@@ -10,7 +10,7 @@ Caper is based on Unix and cloud platform CLIs (`curl`, `gsutil` and `aws`) and
 
 ## Installation
 
-1) Make sure that you have Java (>= 1.8) installed on your system.
+1) Make sure that you have Java (>= 11) installed on your system.
 
 	```bash
 	$ java -version

caper/__init__.py

@@ -2,4 +2,4 @@
 from .caper_runner import CaperRunner
 
 __all__ = ['CaperClient', 'CaperClientSubmit', 'CaperRunner']
-__version__ = '1.6.3'
+__version__ = '1.6.4'

caper/caper_init.py

@@ -108,6 +108,14 @@
 aws-batch-arn=
 aws-region=
 aws-out-dir=
+
+# use this modified cromwell to fix input file localization failures
+# (104 Connection reset by peer)
+# cromwell uses AWS CLI(aws s3 cp)'s native retry feature which is controlled by
+# several environment variables but it doesn't seem to work for some reason
+# this is an adhoc fix to make cromwell retry up to 5 times in the bash script level
+# https://github.com/ENCODE-DCC/cromwell/commit/d16af26483e0019e14d6f8b158eaf64529f57d98
+cromwell=https://storage.googleapis.com/caper-data/cromwell/cromwell-65-d16af26-SNAP.jar
 """
     + CONF_CONTENTS_TMP_DIR
 )

caper/cromwell.py

@@ -37,9 +37,9 @@ class Cromwell:
     """Wraps Cromwell/Womtool.
     """
 
-    DEFAULT_CROMWELL = 'https://github.com/broadinstitute/cromwell/releases/download/59/cromwell-59.jar'
+    DEFAULT_CROMWELL = 'https://github.com/broadinstitute/cromwell/releases/download/65/cromwell-65.jar'
     DEFAULT_WOMTOOL = (
-        'https://github.com/broadinstitute/cromwell/releases/download/59/womtool-59.jar'
+        'https://github.com/broadinstitute/cromwell/releases/download/65/womtool-65.jar'
     )
     DEFAULT_CROMWELL_INSTALL_DIR = '~/.caper/cromwell_jar'
     DEFAULT_WOMTOOL_INSTALL_DIR = '~/.caper/womtool_jar'

scripts/aws_caper_server/create_instance.sh

@@ -224,6 +224,7 @@ aws-batch-arn=$AWS_BATCH_ARN
 aws-region=$AWS_REGION
 aws-out-dir=$AWS_OUT_DIR
 aws-loc-dir=$AWS_LOC_DIR
+cromwell=https://storage.googleapis.com/caper-data/cromwell/cromwell-65-d16af26-SNAP.jar
 # metadata DB
 db=postgresql
 postgresql-db-ip=$POSTGRESQL_DB_IP