Creates new GitHub Action for standalone testing

.github/workflows/standalone.yml

@@ -0,0 +1,63 @@
+name: Empress Standalone CI
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow on pull request and push events, only on the master
+  # branch
+  pull_request:
+    branches: [ master ]
+  push:
+    branches: [ master ]
+
+jobs:
+  # name of job
+  build:
+    # The type of runner that the job will run on (available options are window/macOS/linux)
+    runs-on: ubuntu-latest
+    # we can add more versions of node.js in the future
+    strategy:
+      matrix:
+        # based roughly on https://github.com/conda-incubator/setup-miniconda#example-1-basic-usage
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+
+    # used in McHelper (similar to TRAVIS_PULL_REQUEST variable)
+    env:
+      BRANCH_NUMBER: ${{ github.event.number }}
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # first grab branch from github
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      # https://github.com/conda-incubator/setup-miniconda#example-1-basic-usage
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: empress
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install conda packages
+        shell: bash -l {0}
+        run: conda install flake8 nose
+
+      - name: Install cython & numpy
+        shell: bash -l {0}
+        run: pip install cython "numpy >= 1.12.0"
+
+      - name: Install EMPress
+        shell: bash -l {0}
+        run: pip install -e .[all]
+
+      # tests that don't import qiime2 dependencies
+      - name: Run (non-QIIME 2) Python tests
+        shell: bash -l {0}
+        run: >
+          nosetests
+          ./tests/python/test_cli.py
+          ./tests/python/test_compression_utils.py
+          ./tests/python/test_core.py
+          ./tests/python/test_taxonomy_utils.py
+          ./tests/python/test_tools.py
+          ./tests/python/test_tree.py

README.md

@@ -1,5 +1,6 @@
 # Empress
-[![GitHub Actions CI](https://github.com/biocore/empress/workflows/Empress%20CI/badge.svg)](https://github.com/biocore/empress/actions)
+[![EMPress CI](https://github.com/biocore/empress/actions/workflows/main.yml/badge.svg)](https://github.com/biocore/empress/actions/workflows/main.yml)
+[![EMPress Standalone CI](https://github.com/biocore/empress/actions/workflows/standalone.yml/badge.svg)](https://github.com/biocore/empress/actions/workflows/standalone.yml)
 [![PyPI](https://img.shields.io/pypi/v/empress.svg)](https://pypi.org/project/empress)
 
 <!---Empress Logo--->

tests/python/make-dev-page.py

@@ -5,7 +5,7 @@
 import qiime2 as q2
 import pkg_resources
 
-from test_integration import load_mp_data
+from util import load_mp_data
 
 from bp import parse_newick
 from empress.core import Empress

tests/python/test_cli.py

@@ -1,14 +1,11 @@
 import os
 import unittest
 
-import biom
 from click.testing import CliRunner
 import pandas as pd
-from qiime2 import Artifact
-from skbio.stats.ordination import OrdinationResults
-from skbio.tree import TreeNode
 
 from empress.scripts._cli import empress
+from .util import extract_q2_artifact_to_path
 
 
 def files_present(output_dir):
@@ -21,32 +18,14 @@ class TestCLI(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        q2_tree_loc = "docs/moving-pictures/rooted-tree.qza"
-        q2_table_loc = "docs/moving-pictures/table.qza"
-        q2_sm_loc = "docs/moving-pictures/sample_metadata.tsv"
-        q2_fm_loc = "docs/moving-pictures/taxonomy.qza"
-        q2_pcoa_loc = "docs/moving-pictures/biplot.qza"
+        q2_tree_loc = os.path.abspath("docs/moving-pictures/rooted-tree.qza")
+        q2_table_loc = os.path.abspath("docs/moving-pictures/table.qza")
+        q2_fm_loc = os.path.abspath("docs/moving-pictures/taxonomy.qza")
+        q2_pcoa_loc = os.path.abspath("docs/moving-pictures/biplot.qza")
 
-        cls.tree_loc = "rooted-tree.nwk"
-        cls.table_loc = "table.biom"
-        cls.sm_loc = "sample_metadata.tsv"
-        cls.fm_loc = "taxonomy.tsv"
-        cls.pcoa_loc = "pcoa.txt"
-
-        # convert tree to .nwk
-        nwk_tree = Artifact.load(q2_tree_loc).view(TreeNode)
-
-        # convert table to .biom
-        biom_tbl = Artifact.load(q2_table_loc).view(biom.table.Table)
-
-        # remove comment rows from sample metadata
-        sm = pd.read_csv(q2_sm_loc, sep="\t", index_col=0, skiprows=[1])
-
-        # convert feature metadata to .tsv
-        fm = Artifact.load(q2_fm_loc).view(pd.DataFrame)
-
-        # convert biplot to skbio OrdinationResults
-        pcoa = Artifact.load(q2_pcoa_loc).view(OrdinationResults)
+        q2_sm_loc = os.path.abspath(
+            "docs/moving-pictures/sample_metadata.tsv"
+        )
 
         # create isolated filesystem for tests in this file
         # manually using __enter__ so that we can run all tests and close in
@@ -55,12 +34,22 @@ def setUpClass(cls):
         cls.iso_fs = cls.runner.isolated_filesystem()
         cls.iso_fs.__enter__()
 
-        nwk_tree.write(cls.tree_loc)
-        with biom.util.biom_open(cls.table_loc, "w") as f:
-            biom_tbl.to_hdf5(f, "test")
-        sm.to_csv(cls.sm_loc, index=True, sep="\t")
-        fm.to_csv(cls.fm_loc, index=True, sep="\t")
-        pcoa.write(cls.pcoa_loc)
+        # extract Artifacts to temporary filesystem
+        cls.tree_loc = extract_q2_artifact_to_path("tree", q2_tree_loc,
+                                                   "tree.nwk")
+        cls.table_loc = extract_q2_artifact_to_path("tbl", q2_table_loc,
+                                                    "feature-table.biom")
+        cls.fm_loc = extract_q2_artifact_to_path("fm", q2_fm_loc,
+                                                 "taxonomy.tsv")
+        cls.pcoa_loc = extract_q2_artifact_to_path("pcoa", q2_pcoa_loc,
+                                                   "ordination.txt")
+        # need to re-save sample metadata to remove q2:types row
+        cls.sm_loc = "tmp_sample_metadata.tsv"
+        pd.read_csv(q2_sm_loc, sep="\t", index_col=0, skiprows=[1]).to_csv(
+            cls.sm_loc,
+            sep="\t",
+            index=True
+        )
 
     @classmethod
     def tearDownClass(cls):

tests/python/test_core.py

@@ -18,7 +18,7 @@
 from shutil import rmtree
 import biom
 
-from .test_integration import load_mp_data
+from .util import load_mp_data
 from emperor import Emperor
 from empress import tools
 from empress.core import Empress
@@ -648,21 +648,17 @@ def test_fm_filtering_post_shearing_with_moving_pictures_dataset(self):
         # observe how it is handled in generating a visualization of the
         # moving pictures dataset to verify that #248 does not recur.
         funky_tip = "8406abe6d9a72018bf32d189d1340472"
-        tree, tbl, smd, fmd, pcoa = load_mp_data()
-        # Convert artifacts / metadata objects to "normal" types that we can
-        # pass to Empress
-        bp_tree = from_skbio_treenode(tree.view(TreeNode))
-        tbl_df = tbl.view(biom.Table)
-        pcoa_skbio = pcoa.view(skbio.OrdinationResults)
-        smd_df = smd.to_dataframe()
-        fmd_df = fmd.to_dataframe()
+        tree, tbl, smd_df, fmd_df, pcoa_skbio = load_mp_data(
+            use_artifact_api=False
+        )
+        bp_tree = from_skbio_treenode(tree)
         # Sanity check -- verify that the funky tip we're looking for is
         # actually present in the data. (We haven't actually done anything
         # specific to Empress yet. This just verifies the environment is ok.)
         # https://stackoverflow.com/a/23549599/10730311
         self.assertTrue(funky_tip in fmd_df.index)
         # Generate an Empress visualization using this data
-        viz = Empress(bp_tree, tbl_df, smd_df, feature_metadata=fmd_df,
+        viz = Empress(bp_tree, tbl, smd_df, feature_metadata=fmd_df,
                       ordination=pcoa_skbio, filter_extra_samples=True,
                       shear_to_table=True)
         # Check that tip 8406abe6d9a72018bf32d189d1340472 *isn't* in the tip

tests/python/test_integration.py

@@ -9,47 +9,13 @@
 import os
 import tempfile
 import unittest
-from qiime2 import Artifact, Metadata
 from qiime2.sdk import Results, Visualization
 from qiime2.plugin.testing import TestPluginBase
 
-
-PREFIX_DIR = os.path.join("docs", "moving-pictures")
+from .util import load_mp_data
 
 
-def load_mp_data():
-    """Loads data from the QIIME 2 moving pictures tutorial for visualization.
-
-    It's assumed that this data is already stored in docs/moving-pictures/, aka
-    the PREFIX_DIR global variable set above, which should be located relative
-    to where this function is being run from. If this directory or the data
-    files within it cannot be accessed, this function will (probably) break.
-
-    Returns
-    -------
-    (tree, table, md, fmd, ordination)
-        tree: Artifact with semantic type Phylogeny[Rooted]
-            Phylogenetic tree.
-        table: Artifact with semantic type FeatureTable[Frequency]
-            Feature table.
-        md: Metadata
-            Sample metadata.
-        fmd: Metadata
-            Feature metadata. (Although this is stored in the repository as a
-            FeatureData[Taxonomy] artifact, we transform it to Metadata.)
-        pcoa: Artifact with semantic type PCoAResults
-            Ordination.
-    """
-    tree = Artifact.load(os.path.join(PREFIX_DIR, "rooted-tree.qza"))
-    table = Artifact.load(os.path.join(PREFIX_DIR, "table.qza"))
-    pcoa = Artifact.load(
-        os.path.join(PREFIX_DIR, "unweighted_unifrac_pcoa_results.qza")
-    )
-    md = Metadata.load(os.path.join(PREFIX_DIR, "sample_metadata.tsv"))
-    # We have to transform the taxonomy QZA to Metadata ourselves
-    taxonomy = Artifact.load(os.path.join(PREFIX_DIR, "taxonomy.qza"))
-    fmd = taxonomy.view(Metadata)
-    return tree, table, md, fmd, pcoa
+PREFIX_DIR = os.path.join("docs", "moving-pictures")
 
 
 class TestIntegration(TestPluginBase):

tests/python/util.py

@@ -0,0 +1,112 @@
+import os
+import glob
+import tempfile
+from zipfile import ZipFile
+
+PREFIX_DIR = os.path.join("docs", "moving-pictures")
+
+
+def extract_q2_artifact_to_path(dir_name, artifact_loc, file_name):
+    """Gets relevant data from a Q2 Artifact.
+
+    Figures out the path of an extracted Artifact's data and returns it. This
+    function unzips to the current directory so it should be called in some
+    kind of temporary directory.
+
+    For example: if tmp.qza is a FeatureTable[Frequency] Artifact
+
+        extract_q2_artifact_to_path("table", "tmp.qza", "feature-table.biom")
+
+    would return "table/<UUID>/data/feature-table.biom" and create the
+    "table/<UUID>" subdirectory. We use the glob module to find the filepath
+    because an Artifact unzips to a directory named by a UUID.
+
+    Parameters
+    ----------
+    dir_name: str
+        Name of subdirectory to create with extracted Artifact
+    artifact_loc: str
+        Filepath of Artifact to unzip
+    file_name: str
+        Name of file to search for in data directory
+
+    Returns
+    -------
+    str
+        Filepath of data file in new temporary directory
+    """
+    with ZipFile(artifact_loc, "r") as zip_ref:
+        zip_ref.extractall(dir_name)
+        f = glob.glob(f"{dir_name}/*/data/{file_name}")[0]
+    return f
+
+
+def load_mp_data(use_artifact_api=True):
+    """Loads data from the QIIME 2 moving pictures tutorial for visualization.
+
+    It's assumed that this data is already stored in docs/moving-pictures/, aka
+    the PREFIX_DIR global variable set above, which should be located relative
+    to where this function is being run from. If this directory or the data
+    files within it cannot be accessed, this function will (probably) break.
+
+    Parameters
+    ----------
+    use_artifact_api: bool, optional (default True)
+        If True, this will load the artifacts using the QIIME 2 Artifact API,
+        and the returned objects will have types corresponding to the first
+        listed types (before the | characters) shown below.
+        If False, this will instead load the artifacts without using QIIME 2's
+        APIs; in this case, the returned objects will have types corresponding
+        to the second listed types (after the | characters) shown below.
+
+    Returns
+    -------
+    (tree, table, md, fmd, ordination)
+        tree: qiime2.Artifact | skbio.tree.TreeNode
+            Phylogenetic tree.
+        table: qiime2.Artifact | biom.Table
+            Feature table.
+        md: qiime2.Metadata | pandas.DataFrame
+            Sample metadata.
+        fmd: qiime2.Metadata | pandas.DataFrame
+            Feature metadata. (Although this is stored in the repository as a
+            FeatureData[Taxonomy] artifact, we transform it to Metadata if
+            use_artifact_api is True.)
+        pcoa: qiime2.Artifact | skbio.OrdinationResults
+            Ordination.
+    """
+    q2_tree_loc = os.path.join(PREFIX_DIR, "rooted-tree.qza")
+    q2_table_loc = os.path.join(PREFIX_DIR, "table.qza")
+    q2_pcoa_loc = os.path.join(PREFIX_DIR,
+                               "unweighted_unifrac_pcoa_results.qza")
+    q2_tax_loc = os.path.join(PREFIX_DIR, "taxonomy.qza")
+    md_loc = os.path.join(PREFIX_DIR, "sample_metadata.tsv")
+    if use_artifact_api:
+        from qiime2 import Artifact, Metadata
+
+        tree = Artifact.load(q2_tree_loc)
+        table = Artifact.load(q2_table_loc)
+        pcoa = Artifact.load(q2_pcoa_loc)
+        md = Metadata.load(md_loc)
+        # We have to transform the taxonomy QZA to Metadata ourselves
+        fmd = Artifact.load(q2_tax_loc).view(Metadata)
+    else:
+        import biom
+        import pandas as pd
+        from skbio.stats.ordination import OrdinationResults
+        from skbio.tree import TreeNode
+        with tempfile.TemporaryDirectory() as _tmp:
+            tree_loc = extract_q2_artifact_to_path(_tmp, q2_tree_loc,
+                                                   "tree.nwk")
+            tree = TreeNode.read(tree_loc)
+            tbl_loc = extract_q2_artifact_to_path(_tmp, q2_table_loc,
+                                                  "feature-table.biom")
+            table = biom.load_table(tbl_loc)
+            pcoa_loc = extract_q2_artifact_to_path(_tmp, q2_pcoa_loc,
+                                                   "ordination.txt")
+            pcoa = OrdinationResults.read(pcoa_loc)
+            tax_loc = extract_q2_artifact_to_path(_tmp, q2_tax_loc,
+                                                  "taxonomy.tsv")
+            fmd = pd.read_csv(tax_loc, sep="\t", index_col=0)
+            md = pd.read_csv(md_loc, sep="\t", index_col=0, skiprows=[1])
+    return tree, table, md, fmd, pcoa