First version of dropseq_terra_utils with an email_outputs (#489)

.github/workflows/python-dropseq_terra_utils-conda.yml

@@ -0,0 +1,69 @@
+name: Python Package using Conda
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+    paths:
+      - 'src/python/dropseq_terra_utils/**'
+
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.12'
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        # https://github.com/actions/runner-images/blob/ff9acc6/images/ubuntu/Ubuntu2204-Readme.md#environment-variables
+        echo $CONDA/bin >> $GITHUB_PATH
+    - name: Modify the conda configuration to use conda-forge
+      run: |
+        conda config --add channels conda-forge
+        conda config --remove channels defaults || true
+
+        # There are many issues filed regarding:
+        #   > warning  libmamba Problem type not implemented SOLVER_RULE_STRICT_REPO_PRIORITY
+        # when using the recommended strict channel priority and creating environments from YAML files,
+        # but the FOSS conda/mamba community has not been able to fix the issue.
+        # https://github.com/mamba-org/mamba/issues/2810#issuecomment-1910011988
+        conda config --set channel_priority flexible
+
+        # Disable lock files to avoid bugs in mamba and libmamba
+        # https://mamba.readthedocs.io/en/latest/user_guide/troubleshooting.html#hangs-during-package-installation-on-nfs-network-file-systems
+        # https://github.com/mamba-org/mamba/issues/1993#issuecomment-1268397084
+        echo "use_lockfiles: false" >> ~/.mambarc
+    - name: Install dependencies
+      run: |
+        cd src/python/dropseq_terra_utils
+        conda env update --file environment.yml --name base
+    - name: Lint with flake8
+      run: |
+        cd src/python/dropseq_terra_utils
+
+        # Explicitly using the classic solver to avoid:
+        #   a) "libarchive.so.20: cannot open shared object", and
+        #   b) switching from the miniconda installed in the "ubuntu-latest" image
+        #      https://github.com/actions/runner-images/blob/ff9acc6/images/ubuntu/Ubuntu2204-Readme.md#package-management
+        #   c) upgrading to a working version using the below takes minutes while the tests themselves take seconds
+        #      https://stackoverflow.com/questions/77617946/solve-conda-libmamba-solver-libarchive-so-19-error-after-updating-conda-to-23#answer-78293971
+        conda install --solver=classic --override-channels --channel conda-forge flake8
+
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with unittest
+      run: |
+        cd src/python/dropseq_terra_utils
+        PYTHONPATH=src python -m unittest discover -s tests

.gitignore

@@ -10,3 +10,4 @@ jacoco_reports
 
 .Rproj.user
 .Rhistory
+.DS_Store

src/python/.gitignore

@@ -1 +1,2 @@
 dist
+__pycache__

src/python/dropseq_terra_utils/LICENSE

@@ -0,0 +1,19 @@
+Copyright 2025 Broad Institute
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

src/python/dropseq_terra_utils/README.md

@@ -0,0 +1,13 @@
+# Simple python tools for interacting with workflows in Terra.bio
+
+## Installation
+
+Requires python >= 3.12
+
+```
+pip install 'git+https://github.com/broadinstitute/Drop-seq.git#egg=dropseq_terra_utils&subdirectory=src/python/dropseq_terra_utils'
+```
+
+## Usage
+
+Run `dropseq_terra_utils -h` for usage information.

src/python/dropseq_terra_utils/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# MIT License
+#
+# Copyright 2025 Broad Institute
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/python/dropseq_terra_utils/environment.yml

@@ -0,0 +1,35 @@
+# MIT License
+#
+# Copyright 2025 Broad Institute
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+name: dropseq_terra_utils
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - google-auth
+  - google-cloud-storage
+  - pandas
+  - python=3.12
+  - python-dateutil
+  - pyyaml
+  - requests
+  - urllib3

src/python/dropseq_terra_utils/pyproject.toml

@@ -0,0 +1,52 @@
+# MIT License
+#
+# Copyright 2025 Broad Institute
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+[project]
+name = "dropseq_terra_utils"
+version = "0.0.1"
+dependencies = [
+    "google-auth",
+    "google-cloud-storage",
+    "pandas",
+    "python-dateutil",
+    "pyyaml",
+    "requests",
+    "urllib3",
+]
+description = "Simple python tools for interacting with Drop-seq workflows in Terra.bio."
+readme = "README.md"
+requires-python = ">=3.12"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/broadinstitute/Drop-seq/"
+Issues = "https://github.com/broadinstitute/Drop-seq/issues"
+
+[project.scripts]
+dropseq_terra_utils = "dropseq_terra_utils.cli:main"

src/python/dropseq_terra_utils/src/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# MIT License
+#
+# Copyright 2025 Broad Institute
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/python/dropseq_terra_utils/src/dropseq_terra_utils/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# MIT License
+#
+# Copyright 2025 Broad Institute
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/python/dropseq_terra_utils/src/dropseq_terra_utils/cli.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# MIT License
+#
+# Copyright 2025 Broad Institute
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+Simple python tools for interacting with workflows in Terra.bio.
+"""
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+try:
+    from . import email_outputs
+except ImportError:
+    import email_outputs
+
+# I cannot believe I need to do this to cause logger to write to stderr.
+logging.basicConfig(
+    level=logging.INFO,  # Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[logging.StreamHandler()]  # StreamHandler writes to sys.stderr by default
+)
+logger = logging.getLogger(Path(__file__).stem)
+
+dctLogLevel = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL
+}
+
+
+def main(args=None):
+    parser = argparse.ArgumentParser(prog="dropseq_terra_utils", description=__doc__)
+    parser.add_argument("--log-level", "-l", default="INFO", choices=dctLogLevel.keys(),
+                        help="Set the logging level.  (default: %(default)s)")
+    subparsers = parser.add_subparsers(
+        title="sub-commands",
+        description="valid commands",
+        dest="tool")
+    email_outputs.add_subparser(subparsers)
+
+    if args is None:
+        args = sys.argv[1:]
+    if len(args) == 0:
+        parser.print_help()
+        return 1
+    else:
+        options = parser.parse_args(args)
+        logger.setLevel(dctLogLevel[options.log_level])
+        if options.tool == "email_outputs":
+            return email_outputs.run(options)
+        else:
+            # should be unpossible because parse_args will complain
+            raise ValueError(f"Unrecognized tool: {options.tool}")
+
+
+if __name__ == "__main__":
+    sys.exit(main())