Add support for running Dockstore page URLs and TRS IDs (#5102)

* Add support for running Dockstore page URLs and TRS IDs

* Add missing cast to satisfy typing

* Start porting over to ZIP download based TRS retrieval

* Finish ZIP download and use a workflow cache to avoid responsibility for tmp cleanup

* Fix tests and add always-cached test

* Add session with user agent for TRS requests

* Stop passing extra URL to fetch

* Add missing comma

* Add docstring and boilerplate to new file

* Add missing import

* Move Slurm test to Python 3.9 (latest installed in its Docker images)

* Bump psutil and remove cast MyPy suddenly hates in CI but not on my machine

* Remove some 3.8-isms

* Pyupgrade the new integration file

* Remove more 3.8-isms

* Don't leave behind job store if CWL options don't parse, and add more debugging to tests I can't replicate

* Revert "Remove more 3.8-isms"

This reverts commit d26b6d7.

* Revert "Remove some 3.8-isms"

This reverts commit 9db3484.

* Remove List

* Remove 3.10 type that shouldn't be needed

* Fix error string placeholder

* Take over manual management of CWL job order options and their metavars and help

* Remove now-false note about needing = syntax

* Update the Mesos repo snapshot

* Actually update the Mesos repo snapshot

* Try and fix flaky parallel recursive service test by bounding failure probability

* Eliminate randInt references

* Fix perfect version of service tests

* Fix and test PerfectServiceTest

* Stop printing command line options to toil-cwl-runner standard output

* Fix WDL argument parsing for 3.12 behavior

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

contrib/slurm-test/slurm_test.sh

@@ -12,14 +12,13 @@ docker cp -L sort.py ${LEADER}:/home/admin
 docker cp fileToSort.txt ${LEADER}:/home/admin
 docker cp toil_workflow.py ${LEADER}:/home/admin
 GIT_COMMIT=$(git rev-parse HEAD)
-docker exec ${LEADER} sudo apt install python3-pip -y
-docker exec ${LEADER} pip3 install "git+https://github.com/DataBiosphere/toil.git@${GIT_COMMIT}"
+docker exec ${LEADER} python3.9 -m pip install "git+https://github.com/DataBiosphere/toil.git@${GIT_COMMIT}"
 docker exec ${LEADER} sinfo -N -l
 # Test 1: A really basic workflow to check Slurm is working correctly
-docker exec ${LEADER} python3 /home/admin/toil_workflow.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0 --batchLogsDir ./nonexistent/paths
+docker exec ${LEADER} python3.9 /home/admin/toil_workflow.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0 --batchLogsDir ./nonexistent/paths
 docker cp ${LEADER}:/home/admin/output.txt output_Docker.txt
 # Test 2: Make sure that "sort" workflow runs under slurm
-docker exec ${LEADER} python3 /home/admin/sort.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0
+docker exec ${LEADER} python3.9 /home/admin/sort.py file:my-job-store --batchSystem slurm --disableCaching --retryCount 0
 docker cp ${LEADER}:/home/admin/sortedFile.txt sortedFile.txt
 docker compose stop
 ./check_out.sh

docker/Dockerfile.py

@@ -110,11 +110,12 @@ def heredoc(s):
     # Find a repo with a Mesos build.
     # This one was archived like:
     # mkdir mesos-repo && cd mesos-repo
-    # wget --recursive --restrict-file-names=windows -k --convert-links --no-parent --page-requisites https://rpm.aventer.biz/Ubuntu/ https://www.aventer.biz/assets/support_aventer.asc https://rpm.aventer.biz/README.txt
+    # wget --recursive --restrict-file-names=windows -k --convert-links --no-parent --page-requisites -m https://rpm.aventer.biz/Ubuntu/ https://www.aventer.biz/assets/support_aventer.asc https://rpm.aventer.biz/README.txt
     # ipfs add -r .
-    RUN echo "deb https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmeaErHzK4Dajz2mCMd36eUDQp7GX2bSECVRpGfrqdragR/rpm.aventer.biz/Ubuntu/focal focal main" \
+    # It contains a GPG key that will expire 2026-09-28
+    RUN echo "deb https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmRXnGNiWk523zgNkuamENVkghMJ2zJtinVfgjHbc4Dcpr/rpm.aventer.biz/Ubuntu/focal focal main" \
         > /etc/apt/sources.list.d/mesos.list \
-        && curl https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmeaErHzK4Dajz2mCMd36eUDQp7GX2bSECVRpGfrqdragR/www.aventer.biz/assets/support_aventer.asc | apt-key add -
+        && curl https://public.gi.ucsc.edu/~anovak/outbox/toil/ipfs/QmRXnGNiWk523zgNkuamENVkghMJ2zJtinVfgjHbc4Dcpr/www.aventer.biz/assets/support_aventer.asc | apt-key add -
 
     RUN apt-get -y update --fix-missing && \
         DEBIAN_FRONTEND=noninteractive apt-get -y upgrade && \

requirements.txt

@@ -3,7 +3,7 @@ requests<=2.31.0
 docker>=6.1.0, <8
 urllib3>=1.26.0,<3
 python-dateutil
-psutil >= 3.0.1, < 7
+psutil >= 6.1.0, < 7
 PyPubSub >=4.0.3, <5
 addict>=2.2.1, <2.5
 backports.zoneinfo[tzdata];python_version<"3.9"

src/toil/common.py

@@ -777,29 +777,47 @@ def check_arguments(typ: str) -> None:
 
     # if cwl is set, format the namespace for cwl and check that wdl options are not set on the command line
     if cwl:
-        parser.add_argument("cwltool", type=str, help="CWL file to run.")
-        parser.add_argument(
-            "cwljob",
-            nargs="*",
-            help="Input file or CWL options. If CWL workflow takes an input, "
-            "the name of the input can be used as an option. "
-            'For example: "%(prog)s workflow.cwl --file1 file". '
-            "If an input has the same name as a Toil option, pass '--' before it.",
-        )
+        # So we can manually write out the help for this and the inputs
+        # file/workflow options in the argument parser description, we suppress
+        # help for this option.
+        parser.add_argument("cwltool", metavar="WORKFLOW", type=str, help=SUPPRESS)
+        # We also need a "cwljob" command line argument, holding possibly a
+        # positional input file and possibly a whole string of option flags
+        # only known to the workflow.
+        #
+        # We don't want to try and parse out the positional argument here
+        # since, on Python 3.12, we can grab what's really supposed to be an
+        # argument to a workflow-defined option.
+        #
+        # We don't want to use the undocumented argparse.REMAINDER, since that
+        # will eat any Toil-defined option flags after the first positional
+        # argument.
+        #
+        # So we just use parse_known_args and dump all unknown args into it,
+        # and manually write help text in the argparse description. So don't
+        # define it here.
         check_arguments(typ="cwl")
 
     # if wdl is set, format the namespace for wdl and check that cwl options are not set on the command line
     if wdl:
         parser.add_argument("wdl_uri", type=str, help="WDL document URI")
+        # We want to have an inputs_url that can be either a positional or a flag.
+        # We can't just have them share a single-item dest in Python 3.12;
+        # argparse does not guarantee that will work, and we can get the
+        # positional default value clobbering the flag. See
+        # <https://stackoverflow.com/a/60531838>.
+        # So we make them accumulate to the same list.
+        # Note that we will get a None in the list when there's no positional inputs.
         parser.add_argument(
-            "inputs_uri", type=str, nargs="?", help="WDL input JSON URI"
+            "inputs_uri", type=str, nargs='?', action="append", help="WDL input JSON URI"
         )
         parser.add_argument(
             "--input",
             "--inputs",
             "-i",
             dest="inputs_uri",
             type=str,
+            action="append",
             help="WDL input JSON URI",
         )
         check_arguments(typ="wdl")