Add feature to download google drive datasets

Dockerfile

@@ -46,4 +46,4 @@ RUN echo "Checking autometa entrypoints" \
     && autometa-markers -h > /dev/null \
     && autometa-taxonomy -h > /dev/null \
     && autometa-binning -h > /dev/null \
-    && autometa-unclustered-recruitment -h > /dev/null 
+    && autometa-unclustered-recruitment -h > /dev/null

autometa/binning/recursive_dbscan.py

@@ -931,7 +931,10 @@ def main():
         default="bacteria",
     )
     parser.add_argument(
-        "--verbose", action="store_true", default=False, help="log debug information",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="log debug information",
     )
     args = parser.parse_args()
     kmers_df = kmers.embed(
@@ -940,11 +943,19 @@ def main():
 
     cov_df = pd.read_csv(args.coverages, sep="\t", index_col="contig")
     master_df = pd.merge(
-        kmers_df, cov_df[["coverage"]], how="left", left_index=True, right_index=True,
+        kmers_df,
+        cov_df[["coverage"]],
+        how="left",
+        left_index=True,
+        right_index=True,
     )
     gc_content_df = pd.read_csv(args.gc_content, sep="\t", index_col="contig")
     master_df = pd.merge(
-        master_df, gc_content_df, how="left", left_index=True, right_index=True,
+        master_df,
+        gc_content_df,
+        how="left",
+        left_index=True,
+        right_index=True,
     )
 
     markers_df = load_markers(args.markers)

autometa/binning/summary.py

@@ -303,7 +303,8 @@ def main():
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument(
-        "workspace", help="Path to Autometa results workspace directory",
+        "workspace",
+        help="Path to Autometa results workspace directory",
     )
     parser.add_argument(
         "--write",

autometa/binning/unclustered_recruitment.py

@@ -400,7 +400,9 @@ def get_confidence_filtered_predictions(
         raise NotImplementedError(classifier)
 
     df = pd.DataFrame(
-        predictions, index=test_data.index, columns=train_data.target_names,
+        predictions,
+        index=test_data.index,
+        columns=train_data.target_names,
     )
     # Filter predictions by confidence threshold
     confidence_threshold = num_classifications * confidence

autometa/common/kmers.py

@@ -673,7 +673,10 @@ def main():
         default=False,
     )
     parser.add_argument(
-        "--cpus", help=f"num. processors to use.", default=cpus, type=int,
+        "--cpus",
+        help=f"num. processors to use.",
+        default=cpus,
+        type=int,
     )
     parser.add_argument(
         "--seed",

autometa/common/markers.py

@@ -170,7 +170,11 @@ def get(
 
     if not os.path.exists(out) or not os.path.getsize(out):
         out = hmmer.filter_markers(
-            infpath=scans, outfpath=out, cutoffs=cutoffs, orfs=orfs, force=force,
+            infpath=scans,
+            outfpath=out,
+            cutoffs=cutoffs,
+            orfs=orfs,
+            force=force,
         )
     return load(fpath=out, format=format)
 
@@ -236,7 +240,10 @@ def main():
         type=int,
     )
     parser.add_argument(
-        "--seed", help="Seed to set random state for hmmscan.", default=42, type=int,
+        "--seed",
+        help="Seed to set random state for hmmscan.",
+        default=42,
+        type=int,
     )
     args = parser.parse_args()
 

autometa/common/metagenome.py

@@ -443,7 +443,8 @@ def main():
         "assembly", help="Path to metagenome assembly (nucleotide fasta)."
     )
     parser.add_argument(
-        "out", help="Path to output length-filtered assembly fasta file.",
+        "out",
+        help="Path to output length-filtered assembly fasta file.",
     )
     parser.add_argument(
         "--cutoff",
@@ -471,7 +472,10 @@ def main():
     args = parser.parse_args()
     dirpath = os.path.dirname(os.path.realpath(args.assembly))
     raw_mg = Metagenome(
-        assembly=args.assembly, outdir=dirpath, prot_orfs_fpath="", nucl_orfs_fpath="",
+        assembly=args.assembly,
+        outdir=dirpath,
+        prot_orfs_fpath="",
+        nucl_orfs_fpath="",
     )
 
     filtered_mg = raw_mg.length_filter(

autometa/config/databases.py

@@ -99,7 +99,11 @@ class Databases:
     }
 
     def __init__(
-        self, config=DEFAULT_CONFIG, dryrun=False, nproc=mp.cpu_count(), update=False,
+        self,
+        config=DEFAULT_CONFIG,
+        dryrun=False,
+        nproc=mp.cpu_count(),
+        update=False,
     ):
         """
 
@@ -719,7 +723,9 @@ def main():
         "into default databases directory.",
     )
     parser.add_argument(
-        "--config", help="</path/to/input/database.config>", default=DEFAULT_FPATH,
+        "--config",
+        help="</path/to/input/database.config>",
+        default=DEFAULT_FPATH,
     )
     parser.add_argument(
         "--dryrun",
@@ -769,7 +775,10 @@ def main():
 
     config = get_config(args.config)
     dbs = Databases(
-        config=config, dryrun=args.dryrun, nproc=args.nproc, update=args.update,
+        config=config,
+        dryrun=args.dryrun,
+        nproc=args.nproc,
+        update=args.update,
     )
 
     compare_checksums = False

autometa/config/environ.py

@@ -237,7 +237,9 @@ def get_versions(program=None):
         try:
             return globals()[exe_name]()
         except TypeError:
-            logger.warning(f"{exe_name} not found. This may impact a stage of the Autometa pipeline.")
+            logger.warning(
+                f"{exe_name} not found. This may impact a stage of the Autometa pipeline."
+            )
             return "Not found"
     versions = {}
     executables = find_executables()

autometa/config/project.py

@@ -130,8 +130,7 @@ def new_metagenome_num(self):
             return mg_num
 
     def save(self):
-        """Save project config in project directory
-        """
+        """Save project config in project directory"""
         put_config(self.config, self.config_fpath)
 
     def new_metagenome_directory(self):

autometa/taxonomy/majority_vote.py

@@ -298,7 +298,12 @@ def majority_vote(
         filename, __ = os.path.splitext(os.path.basename(orfs))
         outdir = os.path.dirname(os.path.realpath(out))
         lca_out = os.path.join(outdir, ".".join([filename, "lca.tsv"]))
-    lca_fpath = lca.blast2lca(orfs=orfs, out=lca_out, blast=blast, force=force,)
+    lca_fpath = lca.blast2lca(
+        orfs=orfs,
+        out=lca_out,
+        blast=blast,
+        force=force,
+    )
     # retrieve lca taxids for each contig
     classifications = lca.parse(lca_fpath=lca_fpath, orfs_fpath=orfs)
     # Vote for majority lca taxid from contig lca taxids

autometa/validation/download_dataset.py

@@ -0,0 +1,47 @@
+# in response to issue #110
+# Intended: fetch data similarly to scikit-learn API
+# pulling data from google drive folder with simulated or synthetic communities
+
+# use gdown to download data from google drive to output directory specified by the user
+# create a dictionary of the databases in the google drive
+# allow the user to call them based on size (eg '78', '156'...)
+# allow the user to specify <some/directory>
+# find that corresponding file and download it to <some/directory>
+
+# goal: autometa-download-dataset --community 78 --output <some/directory>
+
+# prepare dependencies
+import gdown
+import argparse
+
+# take in commands that user input
+# including test file for now
+parser = argparse.ArgumentParser(prog='autometa-download-dataset', description='Download a simulated community file from google drive to a specified directory')
+parser.add_argument('--community', 
+                    help='specify a size of simulated community in MB', 
+                    choices=['78', '156', '312', '625', '1250', '2500', '5000', '10000', 'test'], 
+                    required=True)
+parser.add_argument('--output', 
+                    help='specify the directory to download the file', 
+                    required=True)
+args = parser.parse_args()
+
+# provide list of database options as a dictionary with file_ids from google
+simulated = {
+    'test': '1fy3M7RnS_HGSQVKidCy-rAwXuxldyOOv',
+    '78': '15CB8rmQaHTGy7gWtZedfBJkrwr51bb2y',
+    '156': '13bkwFBIUhdWVWlAmVCimDODWF-7tRxgI',
+    '312': '1qyAu-m6NCNuVlDFFC10waOD28j15yfV-',
+    '625': '1FgMXSD50ggu0UJbZd1PM_AvLt-E7gJix',
+    '1250': '1KoxwxBAYcz8Xz9H2v17N9CHOZ-WXWS5m',
+    '2500': '1wKZytjC4zjTuhHdNUyAT6wVbuDDIwk2m',
+    '5000': '1IX6vLfBptPxhL44dLa6jePs-GRw2XJ3S',
+    '10000': '1ON2vxEWC5FHyyPqlfZ0znMgnQ1fTirqG'
+}
+
+# construct file id into a url to put into gdown
+file_id = simulated[args.community]
+url = f'https://drive.google.com/uc?id={file_id}'
+
+# download the specified file with gdown
+gdown.download(url, args.output)

docs/source/conf.py

@@ -20,7 +20,7 @@
 autodoc_mock_imports = ["Bio", "hdbscan", "tsne", "sklearn", "umap", "tqdm"]
 
 # fmt: off
-import parse_argparse  
+import parse_argparse
 
 # -- Project information -----------------------------------------------------
 

requirements.txt

@@ -18,3 +18,4 @@ requests
 umap-learn
 hdbscan
 attrs
+gdown

tests/unit_tests/test_samtools.py

@@ -32,6 +32,7 @@ def test_sort_missing_file():
     with pytest.raises(FileNotFoundError):
         samtools.sort(sam="sam", bam="bam")
 
+
 @pytest.mark.parametrize("cpus", [2.9, -2])
 def test_sort_invalid_cpu_input(cpus):
     with pytest.raises(TypeError):

tests/unit_tests/test_summary.py

@@ -202,7 +202,8 @@ def return_metabin_taxonomies(*args, **kwargs):
 
 @pytest.mark.skip
 def test_get_metabin_taxonomies(
-    mock_rank_taxids, bin_df,
+    mock_rank_taxids,
+    bin_df,
 ):
     mock_ncbi = return_mock_ncbi()
     df = summary.get_metabin_taxonomies(bin_df=bin_df, ncbi=mock_ncbi)

tests/unit_tests/test_unclustered_recruitment.py

@@ -113,7 +113,8 @@ def test_get_kmer_features(kmers, dimensions):
 
 
 def test_get_features_no_taxa(
-    kmers, coverage,
+    kmers,
+    coverage,
 ):
     df = unclustered_recruitment.get_features(kmers=kmers, coverage=coverage)
     assert isinstance(df, pd.DataFrame)

tests/unit_tests/test_vote.py

@@ -91,30 +91,43 @@ def test_add_ranks(ncbi, votes, tmp_path):
 @pytest.mark.skip
 def test_vote_assign(blastp, ncbi_dir, prot_orfs, tmp_path):
     out = tmp_path / "votes.tsv"
-    votes = vote.assign(out=out, prot_orfs=prot_orfs, blast=blastp, ncbi_dir=ncbi_dir,)
+    votes = vote.assign(
+        out=out,
+        prot_orfs=prot_orfs,
+        blast=blastp,
+        ncbi_dir=ncbi_dir,
+    )
     assert isinstance(votes, pd.DataFrame)
     assert votes.index.name == "contig"
     assert "taxid" in votes.columns
 
 
 def test_get(ncbi, votes_fpath):
-    df = vote.get(filepath_or_dataframe=votes_fpath, kingdom="bacteria", ncbi=ncbi,)
+    df = vote.get(
+        filepath_or_dataframe=votes_fpath,
+        kingdom="bacteria",
+        ncbi=ncbi,
+    )
     # canonical ranks should have been added to table if they were not already in place.
     assert df.shape == (2, 8)
 
 
 def test_get_none_recovered(ncbi, votes_fpath):
     with pytest.raises(KeyError):
         vote.get(
-            filepath_or_dataframe=votes_fpath, kingdom="archaea", ncbi=ncbi,
+            filepath_or_dataframe=votes_fpath,
+            kingdom="archaea",
+            ncbi=ncbi,
         )
 
 
 def test_get_empty_votes(ncbi_dir, tmp_path):
     fpath = tmp_path / "votes.tsv"
     with pytest.raises(FileNotFoundError):
         vote.get(
-            filepath_or_dataframe=fpath, kingdom="archaea", ncbi=ncbi_dir,
+            filepath_or_dataframe=fpath,
+            kingdom="archaea",
+            ncbi=ncbi_dir,
         )
 
 
@@ -127,13 +140,19 @@ def return_df(*args, **kwargs):
     monkeypatch.setattr(vote, "add_ranks", return_df, raising=True)
     with pytest.raises(TableFormatError):
         vote.get(
-            filepath_or_dataframe=fpath, kingdom="archaea", ncbi=ncbi,
+            filepath_or_dataframe=fpath,
+            kingdom="archaea",
+            ncbi=ncbi,
         )
 
 
 @pytest.fixture(name="ranks_added_votes", scope="module")
 def fixture_ranks_added_votes(votes_fpath, ncbi):
-    return vote.get(filepath_or_dataframe=votes_fpath, kingdom="bacteria", ncbi=ncbi,)
+    return vote.get(
+        filepath_or_dataframe=votes_fpath,
+        kingdom="bacteria",
+        ncbi=ncbi,
+    )
 
 
 @pytest.mark.parametrize(
@@ -191,7 +210,10 @@ def test_write_ranks_no_taxonomy_columns(tmp_path, votes):
     assembly = dirpath / "assembly.fna"
     with pytest.raises(KeyError):
         vote.write_ranks(
-            taxonomy=votes, assembly=assembly, outdir=dirpath, rank="superkingdom",
+            taxonomy=votes,
+            assembly=assembly,
+            outdir=dirpath,
+            rank="superkingdom",
         )