use env variable to point to bloom/taxclassifier qzvs

.github/workflows/qiita-plugin-ci.yml

@@ -62,7 +62,7 @@ jobs:
 
           # Setting up main qiita conda environment
           conda config --add channels conda-forge
-          conda create -q --yes -n qiita python=3.6 pip==9.0.3 libgfortran numpy nginx cython redis
+          conda create -q --yes -n qiita python=3.9 libgfortran numpy nginx cython redis
           conda activate qiita
           pip install -U pip
           pip install sphinx sphinx-bootstrap-theme nose-timer codecov Click

qp_qiime2/qp_qiime2.py

@@ -194,6 +194,41 @@
 }
 
 
+def _fetch_files(qclient, ainfo):
+    """helper method to fetch all files of an artifact from Qiita main.
+
+    Parameters
+    ----------
+    qclient : qiita_client.QiitaClient
+        The Qiita server client
+    ainfo : ArtifactInfo
+        Information about Qiita artifact
+
+    Returns
+    -------
+    Same as input BUT filepaths are adapated after downloading files from
+    Qiita main to local IF protocol coupling != filesystem. Otherwise, no
+    change occurs.
+    """
+    if qclient._plugincoupling != 'filesystem':
+        if 'files' in ainfo.keys():
+            ainfo['files'] = {
+                filetype: [
+                    {
+                        k: qclient.fetch_file_from_central(v)
+                        if k == 'filepath' else v
+                        for k, v
+                        in file.items()}
+                    for file
+                    in ainfo['files'][filetype]]
+                for filetype
+                in ainfo['files'].keys()
+            }
+        return ainfo
+    else:
+        return ainfo
+
+
 def call_qiime2(qclient, job_id, parameters, out_dir):
     """helper method to call Qiime2
 
@@ -288,6 +323,12 @@ def call_qiime2(qclient, job_id, parameters, out_dir):
                     artifact_id = val
                     ainfo = qclient.get(
                         "/qiita_db/artifacts/%s/" % artifact_id)
+
+                    # If plugin coupling != 'filesystem' download all artifact
+                    # files from Qiita main to here and adapt filepaths
+                    # accordingly.
+                    ainfo = _fetch_files(qclient, ainfo)
+
                     if not q2plugin_is_process and ainfo['analysis'] is None:
                         msg = ('Artifact "%s" is not an analysis '
                                'artifact.' % val)
@@ -323,7 +364,6 @@ def call_qiime2(qclient, job_id, parameters, out_dir):
                             qiita_name['name'], qiita_name['expression'][0])
                     else:
                         artifact_method = qiita_name['name']
-
                 q2inputs[key] = (fpath, artifact_method)
                 # forcing loading of sequences for non_v4_16s
                 if q2method == 'non_v4_16s':
@@ -443,10 +483,12 @@ def call_qiime2(qclient, job_id, parameters, out_dir):
     if q2plugin == 'feature-classifier' and q2method == 'classify_sklearn':
         ainfo = qclient.get("/qiita_db/artifacts/%s/" %
                             parameters['The feature data to be classified.'])
-        biom_fp = ainfo['files']['biom'][0]['filepath']
+        biom_fp = qclient.fetch_file_from_central(
+            ainfo['files']['biom'][0]['filepath'])
         plain_text_fp = None
         if 'plain_text' in ainfo['files']:
-            plain_text_fp = ainfo['files']['plain_text'][0]['filepath']
+            plain_text_fp = qclient.fetch_file_from_central(
+                ainfo['files']['plain_text'][0]['filepath'])
         biom_table = load_table(biom_fp)
         fna_fp = join(out_dir, 'sequences.fna')
         with open(fna_fp, 'w') as f:
@@ -489,15 +531,17 @@ def call_qiime2(qclient, job_id, parameters, out_dir):
         qza = qiime2.Artifact.import_data(
             'FeatureTable[Frequency]', new_biom, 'BIOMV210Format')
         qza.save(new_qza)
-        ftc_fps = [(new_biom, 'biom'), (new_qza, 'qza')]
+        ftc_fps = [(qclient.push_file_to_central(new_biom), 'biom'),
+                   (qclient.push_file_to_central(new_qza), 'qza')]
         if plain_text_fp is not None:
             # if we enter here, it means that the input artifact had a tree
             # (saved as plain_text); thus, we need to make sure we make a copy
             # so we don't move the original file
             bn = basename(plain_text_fp)
             new_tree_fp = join(out_dir, bn)
             copyfile(ainfo['files']['plain_text'][0]['filepath'], new_tree_fp)
-            ftc_fps.append((new_tree_fp, 'plain_text'))
+            ftc_fps.append(
+                (qclient.push_file_to_central(new_tree_fp), 'plain_text'))
         out_info.append(ArtifactInfo(
             'Feature Table with Classification', 'BIOM', ftc_fps))
 
@@ -506,7 +550,8 @@ def call_qiime2(qclient, job_id, parameters, out_dir):
         if isinstance(q2artifact, qiime2.Visualization):
             qzv_fp = q2artifact.save(aout)
             out_info.append(
-                ArtifactInfo(aname, 'q2_visualization', [(qzv_fp, 'qzv')]))
+                ArtifactInfo(aname, 'q2_visualization',
+                             [(qclient.push_file_to_central(qzv_fp), 'qzv')]))
         else:
             qza_fp = q2artifact.save(aout + '.qza')
             q2artifact.export_data(output_dir=aout)
@@ -551,12 +596,15 @@ def call_qiime2(qclient, job_id, parameters, out_dir):
                         out_dir, aout, 'from_%s_%s' % (artifact_id, bn))
                     copyfile(tree_fp, new_tree_fp)
                     ai = ArtifactInfo(aname, 'BIOM', [
-                        (fp, 'biom'),
-                        (new_tree_fp, 'plain_text'),
-                        (qza_fp, 'qza')])
+                        (qclient.push_file_to_central(fp), 'biom'),
+                        (qclient.push_file_to_central(new_tree_fp),
+                         'plain_text'),
+                        (qclient.push_file_to_central(qza_fp), 'qza')])
                 else:
                     ai = ArtifactInfo(
-                        aname, 'BIOM', [(fp, 'biom'), (qza_fp, 'qza')])
+                        aname, 'BIOM',
+                        [(qclient.push_file_to_central(fp), 'biom'),
+                         (qclient.push_file_to_central(qza_fp), 'qza')])
 
             else:
                 qtype = str(q2artifact.type)
@@ -565,7 +613,9 @@ def call_qiime2(qclient, job_id, parameters, out_dir):
                         qtype = 'PCoAResults'
                 atype = Q2_QIITA_SEMANTIC_TYPE[qtype]
                 ai = ArtifactInfo(
-                    aname, atype, [(fp, 'plain_text'), (qza_fp, 'qza')])
+                    aname, atype, [
+                        (qclient.push_file_to_central(fp), 'plain_text'),
+                        (qclient.push_file_to_central(qza_fp), 'qza')])
             out_info.append(ai)
 
     return True, out_info, ""

qp_qiime2/tests/test_qiime2.py

@@ -7,7 +7,7 @@
 # -----------------------------------------------------------------------------
 
 from unittest import main
-from os import remove, stat
+from os import remove, stat, environ
 from shutil import rmtree, copyfile
 from tempfile import mkdtemp
 from json import dumps
@@ -79,7 +79,7 @@ def test_not_analysis_artifact(self):
         self.assertEqual(msg, 'Artifact "5" is not an analysis artifact.')
 
     def test_feature_classifier(self):
-        dbpath = join(self.basedir, '..', '..', 'databases',
+        dbpath = join(environ.get('QP_QIIME2_DBS'),
                       'gg-13-8-99-515-806-nb-classifier.qza')
         original_params = {
             'qp-hide-method': 'classify_sklearn',
@@ -144,6 +144,7 @@ def test_feature_classifier(self):
         biom_fp_new = join(self.basedir, 'support_files', 'deblur.biom')
         copyfile(biom_fp_old, biom_fp_old_bk)
         copyfile(biom_fp_new, biom_fp_old)
+        self.deposite_in_qiita_basedir(biom_fp_old)
 
         self.data['command'] = dumps(
             ['qiime2', qiime2_version, 'Pre-fitted sklearn-based '
@@ -164,6 +165,7 @@ def test_feature_classifier(self):
 
         # returning original biom
         copyfile(biom_fp_old_bk, biom_fp_old)
+        self.deposite_in_qiita_basedir(biom_fp_old)
         obs_files = [ai.files for ai in ainfo]
         obs_artifact_types = [ai.artifact_type for ai in ainfo]
         obs_output_names = [ai.output_name for ai in ainfo]
@@ -1307,7 +1309,7 @@ def test_filter_features(self):
         self.assertEqual(ainfo[0].output_name, 'filtered_table')
 
         # Filter using a qza file
-        qza_path = join(self.basedir, '..', '..', 'filtering', 'blooms-90.qza')
+        qza_path = join(environ.get('QP_QIIME2_FILTER_QZA'), 'blooms-90.qza')
         params.update({
             'Feature metadata used with `where` parameter when selecting '
             'features to retain, or with `exclude_ids` when selecting '

scripts/configure_qiime2

@@ -17,12 +17,14 @@ from qp_qiime2 import plugin
 @click.option('--env-script', prompt='Environment script',
               default='source activate qp-qiime2')
 @click.option('--server-cert', prompt='Server certificate', default='None')
-def config(env_script, server_cert):
+@click.argument('plugincoupling', required=False, default='filesystem')
+def config(env_script, server_cert, plugincoupling):
     """Generates the Qiita configuration files"""
     if server_cert == 'None':
         server_cert = None
     plugin.generate_config(env_script, 'start_qiime2',
-                           server_cert=server_cert)
+                           server_cert=server_cert,
+                           plugin_coupling=plugincoupling)
 
 
 if __name__ == '__main__':