Merge branch 'release/1.13.0'

.circleci/config.yml

@@ -1,10 +1,11 @@
-version: 2 
+version: 2.1
 jobs:
   test:
     docker:
       - image: python:3.7
     steps:
       - checkout
+      - jq/install
       - run:
           name: Install dependencies
           command: pip install -r requirements.txt
@@ -14,8 +15,14 @@ jobs:
       - run:
           name: Link check
           command: cd docs && make linkcheck
+      - run:
+          name: Discourse Topic Check
+          command: |
+            bash discourse-topic-check.sh
+orbs:
+  jq: circleci/jq@2.2.0
 workflows:
-  version: 2
+  version: 2.1
   build:
     jobs:
       - test

base-branch.sh

@@ -0,0 +1,2 @@
+# This file defines the base branch a PR is coming from (for example master or develop)
+baseBranch="develop"

discourse-topic-check.sh

@@ -0,0 +1,60 @@
+#
+#    Copyright 2022 OICR and UCSC
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# This script determines if all changed files rst files in a PR have a discourse topic
+source base-branch.sh
+
+# Determines if a file has a discourse topic
+
+function containsDiscourseTopic {
+  grep -A1 "^.. discourse::" $fileToCheck | tail -n1 | grep -E "^( )*:topic_identifier:( )*[0-9]+" > /dev/null
+  if [ $? != 0 ]
+  then
+    echo "${fileToCheck} does not have a discourse topic"
+    return 1
+  else
+    return 0
+  fi
+}
+
+RETURN_VALUE=0
+DOES_NOT_REQUIRE_DISCOURSE_TOPIC=no-discourse-topic-required.txt
+
+if [[ -n $CIRCLE_PULL_REQUEST ]]
+then
+  # This step requires the variable CIRCLE_PULL_REQUEST to be set to the URL of the PR,
+  # this is done automatically in CircleCI.
+  # An example value of CIRCLE_PULL_REQUEST is https://github.com/dockstore/dockstore-documentation/pull/209
+  pr=$(echo $CIRCLE_PULL_REQUEST | sed 's+https://github.com+https://api.github.com/repos+' | sed 's/pull/pulls/')
+  baseBranch="$(curl -s $pr | jq -r '.base.ref')"
+fi
+
+for file in $(git diff --name-only "$baseBranch".. | grep -E "*\.rst" | grep -Fvxf $DOES_NOT_REQUIRE_DISCOURSE_TOPIC)
+do
+  fileToCheck=$file
+  if ! containsDiscourseTopic
+  then
+    RETURN_VALUE=1
+  fi
+done
+
+if [ $RETURN_VALUE != 0 ]
+then
+  echo "If your files do not require a discourse topic you can add them to ${DOES_NOT_REQUIRE_DISCOURSE_TOPIC}"
+else
+  echo "All of the modified files have discourse topics"
+fi
+
+exit $RETURN_VALUE

docs/Makefile

@@ -13,7 +13,7 @@ help:
 
 glossary:
 	echo "\033[1mGenerating dictionary.rst...\033[0m"
-	python _attic/glossary_generator.py
+	python3 _attic/glossary_generator.py
 
 .PHONY: help Makefile
 

docs/_attic/glossary_entries.py

@@ -22,6 +22,12 @@
 AGC = GlossEntry("AGC",
 	acronym_full="[Amazon Genomics CLI]")
 
+AbsolutePath = GlossEntry("absolute path",
+	definition="A path that starts with the character ``/`` and contains the full set of directories necessary to resolve a file, starting from the root directory of the repository or filesystem. For example: ``/Dockstore.cwl`` or ``/bin/sh``",
+	furtherreading="",
+	institute="",
+	pronunciation='')
+
 AmazonGenomicsCLI = GlossEntry("Amazon Genomics CLI", 
 	definition="A [CLI]-based tool that supports launching bioinformatics-related workflows on [AWS] cloud infrastructure. The [Dockstore CLI] can launch workflows on AWS using Amazon Genomics CLI's [WES] implementation.", 
 	furtherreading="https://aws.amazon.com/blogs/industries/announcing-amazon-genomics-cli-preview/", 
@@ -193,6 +199,13 @@
 	institute="", 
 	pronunciation="")
 
+DRS = GlossEntry("DRS",
+        acronym_full="Data Repository Service",
+        definition="A standardized [API], created by the [GA4GH] Cloud Work Stream, that provides portable access to repositories of data resources.",
+        furtherreading="https://github.com/ga4gh/data-repository-service-schemas",
+        institute="GA4GH",
+        pronunciation='"derse", rhymes with verse')
+
 DSIAfrica = GlossEntry("DS-I Africa", 
 	acronym_full="Data Science for health discovery and Innovation in Africa", 
 	definition="An [NIH] initiative to leverage data science to address the African continent's public health needs.", 
@@ -251,7 +264,7 @@
 
 Galaxy = GlossEntry("Galaxy", 
 	acronym_full="", 
-	definition="An open-source platform that uses [FAIR] principles, most well-known for its web-based UI used to create and run a variety of bioinformatics tools.", 
+	definition="An open-source platform that uses [FAIR] principles, most well-known for its web-based UI used to create and run a variety of bioinformatics tools. A Galaxy `instance` is a running Galaxy interface/server that can be used to create and execute tools and workflows.",
 	furtherreading="https://galaxyproject.org/", 
 	institute="", 
 	pronunciation='')
@@ -475,6 +488,20 @@
 	pronunciation='', 
 	seealso="[spot instance]")
 
+PrimaryDescriptorFile = GlossEntry("primary descriptor file",
+	acronym_full="",
+	definition="The [descriptor file] that provides the overall description of a workflow or tool, which Dockstore processes first when the workflow or tool is registered.",
+	furtherreading="",
+	institute="",
+	pronunciation='')
+
+SecondaryDescriptorFile = GlossEntry("secondary descriptor file",
+	acronym_full="",
+	definition="An ancillary [descriptor file], referenced by the [primary descriptor file] or another secondary descriptor file, that describes part of a workflow or tool.",
+	furtherreading="",
+	institute="",
+	pronunciation='')
+
 SevenBridges = GlossEntry("Seven Bridges", 
 	acronym_full="", 
 	definition="A cloud-based workflow execution platform developed by Seven Bridges Genomics. Seven Bridges supports the execution of [CWL] workflows and features a graph-based GUI to make workflow development easier. The computational backend of a Seven Bridges workspace can be selected by the user, with both [GCP] and [AWS] being supported. Dockstore supports directly importing [CWL] workflows into a Seven Bridges workspace. Seven Bridges is part of the [BioData Catalyst] consortium.", 
@@ -514,6 +541,13 @@
 	pronunciation='', 
 	seealso="[workflow]")
 
+TRS = GlossEntry("TRS",
+        acronym_full="Tool Registry Service",
+        definition="A standardized [API], created by the [GA4GH] Cloud Work Stream, that provides portable access to a registry of tools, workflows, and associated files.  Every resource in a TRS registry has a public ID that can be used to retrieve it.  Dockstore `provides <https://dockstore.org/api/static/swagger-ui/index.html#/GA4GHV20>`__ a TRS interface.",
+        furtherreading="https://ga4gh.github.io/tool-registry-service-schemas/",
+        institute="GA4GH",
+        pronunciation='"terse", rhymes with verse')
+
 UCSC = GlossEntry("UCSC", 
 	acronym_full="University of California, Santa Cruz", 
 	definition="A public university located in Santa Cruz that is focused on undergraduate and graduate education and research. The Genomics Institute, a branch of UCSC's engineering department, is one of the two institutes involved in the development of Dockstore, the other being [OICR].", 

docs/_attic/glossary_entries_list_dynamic.txt

@@ -1,5 +1,6 @@
 DO NOT EDIT THIS FILE. This file is autogenerated from glossary_generator.py, update that instead.
 .dockstore.yml
+absolute path
 AGC
 Amazon Genomics CLI
 AnVIL Project
@@ -27,6 +28,7 @@ Dockerfile
 Dockstore CLI
 Dockstore GitHub App
 DOI
+DRS
 DS-I Africa
 EC2
 egress
@@ -66,12 +68,15 @@ ORCID
 organization
 parameter file
 preemptible
+primary descriptor file
+secondary descriptor file
 Seven Bridges
 Spot Instance
 Task Execution Service
 Terra
 TES
 tool
+TRS
 UCSC
 VM
 WDL

docs/advanced-topics/docker-alternatives.rst

@@ -5,6 +5,9 @@ Docker Alternatives
 ===================
 
 In some situations using Docker may be impractical because it requires all users to have root access.
+Several alternatives have been developed to make it possible to run rootless containers, including
+`Singularity <https://sylabs.io/docs/>`_ and 
+`rootless Docker <https://web.archive.org/web/20210625103301/https://www.docker.com/blog/experimenting-with-rootless-docker/>`_.
 While Dockstore uses Docker by default, if necessary it may be possible to run your workflows with one
 of these alternatives. Because the call to Docker or an alternative is made by the workflow runner, usually cwltool
 or Cromwell, and not Dockstore directly, the difficulty of configuring a Docker alternative depends on the workflow

docs/advanced-topics/dockstore-cli/advanced-features.rst

@@ -253,8 +253,7 @@ By default, the Dockstore CLI launches CWL tools/workflows using
 However, we have an experimental integration with other launchers such
 as: -
 `cwl-runner <https://www.commonwl.org/v1.0/CommandLineTool.html#Executing_CWL_documents_as_scripts>`__
-- `Cromwell <https://cromwell.readthedocs.io/en/stable/>`__ (For
-Dockstore 1.6.0+)
+- `Cromwell <https://cromwell.readthedocs.io/en/stable/>`__
 
 Keep in mind that there are a few differences in how locked-down the
 Docker execution environments are between the launchers. So a workflow
@@ -282,13 +281,11 @@ following to your ``~/.dockstore/config``:
 
     cwlrunner: cwl-runner
 
-.. _cromwell-beta:
+.. _cromwell:
 
-Cromwell (Beta)
+Cromwell
 ~~~~~~~~~~~~~~~
 
-    For Dockstore 1.6.0+
-
 You can launch CWL tools/workflows using Cromwell by adding the
 following to your ``~/.dockstore/config``:
 
@@ -340,7 +337,7 @@ Additionally, you can override the Cromwell version in your
 
 ::
 
-    cromwell-version = 76
+    cromwell-version = 84
 
 The Dockstore CLI will attempt to download the version of Cromwell JAR file you specify from the `Cromwell
 download area <https://github.com/broadinstitute/cromwell/releases/>`__ to

docs/advanced-topics/dockstore-cli/yaml-command-line-validator-tool.rst

@@ -0,0 +1,118 @@
+Dockstore Yaml Command Line Validator Tool
+==========================================
+
+Description
+-----------------
+
+The Dockstore Yaml Command Line Validator Tool is used to verify that a :ref:`dict .dockstore.yml` file is valid for use in Dockstore and that all referenced files are present.
+
+The Dockstore Yaml Command Line Validator Tool is particularly helpful when you are trying to sync a GitHub repository with Dockstore by adding a :ref:`dict .dockstore.yml` to the root or ``.github`` directory of the repository. As you can verify that :ref:`dict .dockstore.yml` is valid before pushing, and it can help you to determine why your repository is not syncing with Dockstore.
+
+The Dockstore Yaml Command Line Validator Tool will first determine if :ref:`dict .dockstore.yml` is a valid yaml file and display the following if :ref:`dict .dockstore.yml` is a valid yaml file (but not necessarily valid for use in Dockstore),
+
+::
+
+  path/to/.dockstore.yml is a valid yaml file
+
+
+If :ref:`dict .dockstore.yml` is not a valid yaml file the following will be displayed along with an error message explaining why :ref:`dict .dockstore.yml` is not a valid yaml file,
+
+::
+
+  path/to/.dockstore.yml is not a valid yaml file
+
+
+
+Next, the Dockstore Yaml Command Line Validator Tool will determine if :ref:`dict .dockstore.yml` is valid for use in Dockstore, if it is it will display this message,
+
+::
+
+  path/to/.dockstore.yml is a valid dockstore yaml file and all required files are present
+  
+If :ref:`dict .dockstore.yml` is not a valid for use in Dockstore the Dockstore Yaml Command Line Validator Tool will display a helpful error message, such as,
+
+::
+
+  Your file structure has the following errors:
+  path/to/dockstore.cwl.json does not exist
+
+or,
+
+::
+
+  path/to/.dockstore.yml has the following errors:
+  Unknown property: 'primaryDescriptorath'. Did you mean: 'primaryDescriptorPath'?
+
+Usage
+-----------------
+
+The Dockstore Yaml Command Line Validator Tool can be used with the following command in the :ref:`dict Dockstore CLI`,
+
+::
+
+  dockstore yaml validate --path directory/of/.dockstore.yml
+
+Path Parameter
+```````````````
+
+The ``--path`` parameter must be set to the directory that contains :ref:`dict .dockstore.yml`, but must not include :ref:`dict .dockstore.yml`.
+For example,
+
+::
+
+  path/to/my/awesome/workflow
+
+is acceptable, however
+
+::
+
+  path/to/my/awesome/workflow/.dockstore.yml
+
+is not acceptable.
+
+The ``--path`` parameter can be either an absolute or relative directory. Therefore, the following are valid uses of the Dockstore Yaml Command Line Validator Tool,
+
+::
+
+  dockstore yaml validate --path .
+
+
+::
+
+  dockstore yaml validate --path directory/of/.dockstore.yml
+
+::
+
+  dockstore yaml validate --path ../../path/to/service
+
+::
+
+  dockstore yaml validate --path ~/path/to/workflow
+
+::
+
+  dockstore yaml validate --path /usr/jdoe/dockstore/workflow
+
+
+Please note that all files referenced in :ref:`dict .dockstore.yml`, are checked relative to the path parameter, unless the path parameter ends in ``.github``, in this case all files referenced in  :ref:`dict .dockstore.yml` are checked relative to the parent of the path parameter.
+
+For example if ``./my/awesome/workflow/.dockstore.yml`` contained the following,
+
+::
+
+  testParameterFiles:
+    - /dockstore.wdl.json
+
+and you selected ``./my/awesome/workflow`` as the path parameter, then the Dockstore Yaml Command Line Validator Tool would check that the file ``./my/awesome/workflow/dockstore.wdl.json`` exists.
+
+However, if ``./my/fantastic/workflow/.github/.dockstore.yml`` contained the following,
+
+::
+
+  testParameterFiles:
+    - /workflow.cwl.json
+
+and you selected ``./my/fantastic/workflow/.github`` as the path parameter, then the Dockstore Yaml Command Line Validator Tool would check that the file ``./my/fantastic/workflow/workflow.cwl.json`` exists.
+
+.. discourse::
+    :topic_identifier: 5577

docs/advanced-topics/naming-workflows.rst

@@ -5,7 +5,7 @@ each workflow on Dockstore has a unique identifier in the form of a path. This p
 the Git repository that the workflow comes from. There are four components to a path, but only
 three are required. It has the following structure:
 
-``<sourceControl>/<organization name>/<repository name>/<optional workflow name>:<version name>``
+``<source-control>/<organization-name>/<repository-name>/<optional-workflow-name>:<version-name>``
 
 Why not simply use a number to identify the workflow? With a path like that shown above, users
 can quickly understand the purpose of a workflow along with where it came from.
@@ -14,8 +14,10 @@ Ex. If I had a GitHub repository called BAMstats that existed in the OICR organi
 
 ``github.com/OICR/BAMstats``
 
-The final optional component for the workflow path is the workflow name. This is a user defined
-string that will be appended to the end of the required workflow path. It is useful in two situations:
+The final optional component for the workflow path is the workflow name, which is a user-defined
+string that will be appended to the end of the required workflow path. It can contain letters, numbers, internal hyphens, and internal underscores, but no spaces or other characters.
+
+A workflow name is useful in two situations:
 
 1) The name of the repository doesn't represent the workflow, or
 2) The repository contains multiple workflows
@@ -24,4 +26,4 @@ Using the previous example, we could set the workflow name to ``coverage``. Our
 
 ``github.com/OICR/BAMstats/coverage``
 
-If we set the workflow name, we must include it in our path when referencing the workflow. You also should be aware of a workflow's name when it comes to migrating a workflow registered via legacy methods to GitHub App registration methods. During the migration process, be sure to include the workflow's name as a field on your .dockstore.yml file.
+If we set the workflow name, we must include it in our path when referencing the workflow. You also should be aware of a workflow's name when it comes to migrating a workflow registered via legacy methods to GitHub App registration methods. During the migration process, be sure to include the workflow's name as a field on your .dockstore.yml file.