From 66265df1966b64081279050f2db91f24aedd81e2 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 16 Jun 2017 10:37:42 -0600 Subject: [PATCH 01/18] nose-timer to pip command --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d733d36cd..65d0d7086 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,11 +23,11 @@ install: # install a few of the dependencies that pip would otherwise try to install # when intalling scikit-bio - travis_retry conda create --yes -n qiita python=$PYTHON_VERSION pip nose flake8 - pyzmq networkx pyparsing natsort mock future libgfortran seaborn nose-timer + pyzmq networkx pyparsing natsort mock future libgfortran seaborn 'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1' - source activate qiita - pip install -U pip - - pip install sphinx sphinx-bootstrap-theme coveralls 'ipython[all]==2.4.1' + - pip install sphinx sphinx-bootstrap-theme coveralls 'ipython[all]==2.4.1' nose-timer - travis_retry pip install . --process-dependency-links - 'echo "backend: Agg" > matplotlibrc' # Install the biom plugin so we can run the analysis tests From 304e2e457bc1fcaf68abd81ed845c913ac412abc Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sun, 18 Jun 2017 06:20:37 -0600 Subject: [PATCH 02/18] improving study listing // artifact info --- qiita_db/artifact.py | 68 +++++++ qiita_db/test/test_artifact.py | 31 ++++ qiita_db/test/test_util.py | 49 +---- qiita_db/util.py | 173 +----------------- qiita_pet/handlers/api_proxy/__init__.py | 5 +- qiita_pet/handlers/api_proxy/artifact.py | 39 ++++ .../handlers/api_proxy/tests/test_artifact.py | 43 ++++- qiita_pet/handlers/study_handlers/__init__.py | 4 +- qiita_pet/handlers/study_handlers/artifact.py | 12 +- .../study_handlers/tests/test_artifact.py | 29 ++- .../tests/test_listing_handlers.py | 26 +-- qiita_pet/static/css/style.css | 14 +- qiita_pet/templates/list_studies.html | 137 ++++++++------ qiita_pet/webserver.py | 7 +- 14 files changed, 330 insertions(+), 307 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 3113a4bd7..9a3d0ac03 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -1181,6 +1181,74 @@ def analysis(self): res = qdb.sql_connection.TRN.execute_fetchindex() return qdb.analysis.Analysis(res[0][0]) if res else None + @property + def biom_info(self): + """Returns processing information about the bioms in the artifact + + Returns + ------- + dict or None + The info of the bioms if artifact_type is BIOM or None if not. + """ + if self.artifact_type != 'BIOM': + return None + files = [(fid, fp)for fid, fp, fpt in self.filepaths if fpt == 'biom'] + if not files: + return None + + parameters = {} + with qdb.sql_connection.TRN: + # getting target_subfragment + sql = """ + SELECT DISTINCT target_subfragment + FROM qiita.prep_%s + WHERE EXISTS (SELECT 1 + FROM information_schema.columns + WHERE table_name='prep_%s' + AND column_name='target_subfragment')""" + pt_ids = [[pt.id, pt.id] for pt in self.prep_templates] + qdb.sql_connection.TRN.add(sql, pt_ids, many=True) + target_subfragment = qdb.sql_connection.TRN.execute_fetchflatten() + + # getting processing_parameters + pp = self.processing_parameters + if pp is not None: + parameters = pp.values + parents = self.parents + if bool(parents): + # [0] an artifact can only have one processing parent + parent = parents[0] + ppp = parent.processing_parameters.command.name + # obtaining all processing parameters so we can then + # match, the parents processing params + sql_params = """ + SELECT parameter_set_name, array_agg(ps) AS param_set + FROM qiita.default_parameter_set, + json_each_text(parameter_set) ps + GROUP BY parameter_set_name""" + qdb.sql_connection.TRN.add(sql_params) + params = {pname: eval(params) for pname, params + in qdb.sql_connection.TRN.execute_fetchindex()} + tpppv = {'(%s,%s)' % (k, v) + for k, v in viewitems( + parent.processing_parameters.values)} + pppv = sorted([ + [k, len(tpppv & v)] for k, v in viewitems(params)], + key=lambda x: x[1])[-1][0] + else: + ppp = '' + pppv = '' + + algorithm = '%s | %s (%s)' % ( + pp.command.name, ppp, pppv) + else: + algorithm = 'N/A' + + return {'target_subfragment': target_subfragment, 'name': self.name, + 'data_type': self.data_type, 'timestamp': self.timestamp, + 'parameters': parameters, 'algorithm': algorithm, + 'files': files} + def jobs(self, cmd=None, status=None): """Jobs that used this artifact as input diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 0f51c498c..6d3f29767 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -407,6 +407,37 @@ def test_analysis(self): qdb.analysis.Analysis(1)) self.assertIsNone(qdb.artifact.Artifact(1).analysis) + def test_biom_info(self): + # testing not biom + self.assertIsNone(qdb.artifact.Artifact(1).biom_info) + self.assertIsNone(qdb.artifact.Artifact(2).biom_info) + # test no biom // empty analysis + self.assertIsNone(qdb.artifact.Artifact(7).biom_info) + # regular processed file + bdir = qdb.util.get_db_files_base_dir() + obs = qdb.artifact.Artifact(4).biom_info + exp = { + 'files': [(9, join(bdir, 'processed_data/1_study_1001_closed' + '_reference_otu_table.biom'))], + 'target_subfragment': ['V4'], 'parameters': { + 'reference': 1, 'similarity': 0.97, 'sortmerna_e_value': 1, + 'sortmerna_max_pos': 10000, 'input_data': 2, 'threads': 1, + 'sortmerna_coverage': 0.97}, + 'algorithm': ( + 'Pick closed-reference OTUs | Split libraries FASTQ ' + '(Defaults with reverse complement mapping file barcodes)'), + 'timestamp': datetime(2012, 10, 2, 17, 30), + 'data_type': '18S', 'name': 'BIOM'} + self.assertEqual(obs, exp) + # analysis + obs = qdb.artifact.Artifact(8).biom_info + exp = { + 'files': [(15, join(bdir, 'analysis/1_analysis_18S.biom'))], + 'target_subfragment': [], 'parameters': {}, 'algorithm': 'N/A', + 'timestamp': datetime(2017, 6, 16, 15, 28, 28, 943120), + 'data_type': '18S', 'name': 'noname'} + self.assertEqual(obs, exp) + def test_jobs(self): obs = qdb.artifact.Artifact(1).jobs() exp = [ diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 5e609d3ee..288689442 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -807,7 +807,6 @@ def test_generate_study_list(self): 'shared': [('shared@foo.bar', 'Shared')], 'pi': ('PI_dude@foo.bar', 'PIDude'), 'status': 'private', - 'proc_data_info': [], 'publication_doi': ['10.100/123456', '10.100/7891011'], 'publication_pid': ['123456', '7891011'], 'study_abstract': ( @@ -832,7 +831,7 @@ def test_generate_study_list(self): 'ebi_submission_status': 'not submitted', 'publication_pid': [], 'study_abstract': 'Some abstract goes here', 'pi': ('lab_dude@foo.bar', 'LabDude'), 'status': 'sandbox', - 'proc_data_info': [], 'study_tags': None, 'shared': [], + 'study_tags': None, 'shared': [], 'publication_doi': [], 'study_id': new_study.id, 'ebi_study_accession': None, 'study_title': 'test_study_1', 'number_samples_collected': 0}] @@ -841,55 +840,9 @@ def test_generate_study_list(self): qdb.artifact.Artifact(4).visibility = 'public' exp_info[0]['status'] = 'public' - exp_info[0]['proc_data_info'] = [ - {'data_type': '18S', 'name': 'BIOM', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 4, 'processed_date': '2012-10-02 17:30:00', - 'params': {'similarity': 0.97, 'reference_name': 'Greengenes', - 'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000, - 'threads': 1, u'sortmerna_coverage': 0.97, - 'reference_version': '13_8'}}, - {'data_type': '18S', 'name': 'BIOM', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 5, 'processed_date': '2012-10-02 17:30:00', - 'params': {'similarity': 0.97, 'reference_name': 'Greengenes', - 'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000, - 'threads': 1, 'sortmerna_coverage': 0.97, - 'reference_version': '13_8'}}, - {'data_type': '16S', 'name': 'BIOM', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 6, 'processed_date': '2012-10-02 17:30:00', - 'params': {'similarity': 0.97, 'reference_name': 'Silva', - 'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000, - 'threads': 1, 'sortmerna_coverage': 0.97, - 'reference_version': 'test'}}] obs_info = qdb.util.generate_study_list([1, 2, 3, 4], True) self.assertEqual(obs_info, exp_info) - exp_info[0]['proc_data_info'] = [ - {'data_type': '18S', 'name': 'BIOM', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 4, 'processed_date': '2012-10-02 17:30:00', - 'params': {'similarity': 0.97, 'reference_name': 'Greengenes', - 'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000, - 'threads': 1, 'sortmerna_coverage': 0.97, - 'reference_version': '13_8'}}, - {'data_type': '18S', 'name': 'BIOM', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 5, 'processed_date': '2012-10-02 17:30:00', - 'params': {'similarity': 0.97, 'reference_name': 'Greengenes', - 'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000, - 'threads': 1, 'sortmerna_coverage': 0.97, - 'reference_version': '13_8'}}, - {'data_type': '16S', 'name': 'BIOM', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 6, 'processed_date': '2012-10-02 17:30:00', - 'params': {'similarity': 0.97, 'reference_name': 'Silva', - 'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000, - 'threads': 1, 'sortmerna_coverage': 0.97, - 'reference_version': 'test'}}, - {'processed_date': '2012-10-02 17:30:00', 'pid': 7, 'name': 'BIOM', - 'data_type': '16S'}] obs_info = qdb.util.generate_study_list([1, 2, 3, 4], False) self.assertEqual(obs_info, exp_info) diff --git a/qiita_db/util.py b/qiita_db/util.py index 4fce2cdc7..bb1d974fb 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -1211,7 +1211,7 @@ def generate_study_list(study_ids, public_only=False): - the total number of samples collected by counting sample_ids (SELECT COUNT(sample_id) FROM qiita.study_sample WHERE study_id=qiita.study.study_id) - AS number_samples_collected, + AS number_samples_collected] - all the BIOM artifact_ids sorted by artifact_id that belong to the study (SELECT array_agg(artifact_id ORDER BY artifact_id) FROM qiita.study_artifact @@ -1219,60 +1219,6 @@ def generate_study_list(study_ids, public_only=False): LEFT JOIN qiita.artifact_type USING (artifact_type_id) WHERE artifact_type='BIOM' AND study_id = qiita.study.study_id) AS artifact_biom_ids, - - all the BIOM data_types sorted by artifact_id that belong to the study - (SELECT array_agg(data_type ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.data_type USING (data_type_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_dts, - - all the BIOM parameters sorted by artifact_id that belong to the study - (SELECT array_agg(command_parameters ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) - AS artifact_biom_params, - - all the BIOM command_ids sorted by artifact_id that belong to the study, - (SELECT array_agg(command_id ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) - AS artifact_biom_cmd, - - all the BIOM timestamps sorted by artifact_id that belong to the study - (SELECT array_agg(generated_timestamp ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_ts, - - all the BIOM names sorted by artifact_id that belong to the study - (SELECT array_agg(name ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_name, - - all the BIOM visibility sorted by artifact_id that belong to the study - (SELECT array_agg(visibility ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - LEFT JOIN qiita.visibility USING (visibility_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_vis, - - all the visibilities of all artifacts that belong to the study - (SELECT array_agg(DISTINCT visibility) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - LEFT JOIN qiita.visibility USING (visibility_id) - WHERE study_id = qiita.study.study_id) - AS artifacts_visibility, - all the publications that belong to the study (SELECT array_agg((publication, is_doi))) FROM qiita.study_publication @@ -1304,53 +1250,6 @@ def generate_study_list(study_ids, public_only=False): LEFT JOIN qiita.artifact_type USING (artifact_type_id) WHERE artifact_type='BIOM' AND study_id = qiita.study.study_id) AS artifact_biom_ids, - (SELECT array_agg(data_type ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.data_type USING (data_type_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_dts, - (SELECT array_agg(command_parameters ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) - AS artifact_biom_params, - (SELECT array_agg(command_id ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) - AS artifact_biom_cmd, - (SELECT array_agg(generated_timestamp ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_ts, - (SELECT array_agg(name ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_name, - (SELECT array_agg(visibility ORDER BY artifact_id) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - LEFT JOIN qiita.visibility USING (visibility_id) - WHERE artifact_type='BIOM' AND - study_id = qiita.study.study_id) AS artifact_biom_vis, - (SELECT array_agg(DISTINCT visibility) - FROM qiita.study_artifact - LEFT JOIN qiita.artifact USING (artifact_id) - LEFT JOIN qiita.artifact_type USING (artifact_type_id) - LEFT JOIN qiita.visibility USING (visibility_id) - WHERE study_id = qiita.study.study_id) - AS artifacts_visibility, (SELECT array_agg(row_to_json((publication, is_doi), true)) FROM qiita.study_publication WHERE study_id=qiita.study.study_id) AS publications, @@ -1368,8 +1267,6 @@ def generate_study_list(study_ids, public_only=False): WHERE study_id IN %s ORDER BY study_id""" qdb.sql_connection.TRN.add(sql, [tuple(study_ids)]) infolist = [] - refs = {} - commands = {} for info in qdb.sql_connection.TRN.execute_fetchindex(): info = dict(info) @@ -1387,12 +1284,6 @@ def generate_study_list(study_ids, public_only=False): info['publication_pid'].append(pub) del info['publications'] - # visibility - # infer_status expects a list of list of str - iav = info['artifacts_visibility'] - del info['artifacts_visibility'] - info["status"] = infer_status([[s] for s in iav] if iav else []) - # pi info info["pi"] = (info['pi_email'], info['pi_name']) del info["pi_email"] @@ -1409,76 +1300,20 @@ def generate_study_list(study_ids, public_only=False): del info["shared_with_name"] del info["shared_with_email"] - info['proc_data_info'] = [] - if info['artifact_biom_ids']: - to_loop = zip( - info['artifact_biom_ids'], info['artifact_biom_dts'], - info['artifact_biom_ts'], info['artifact_biom_params'], - info['artifact_biom_cmd'], info['artifact_biom_vis'], - info['artifact_biom_name']) - for artifact_id, dt, ts, params, cmd, vis, name in to_loop: - if public_only and vis != 'public': - continue - proc_info = {'processed_date': str(ts)} - proc_info['pid'] = artifact_id - proc_info['data_type'] = dt - proc_info['name'] = name - - # if cmd exists then we can get its parameters - if cmd is not None: - # making sure that the command is only queried once - if cmd not in commands: - c = qdb.software.Command(cmd) - commands[cmd] = { - # remove artifacts from parameters - 'del_keys': [k for k, v in viewitems( - c.parameters) if v[0] == 'artifact'], - 'sfwn': c.software.name, - 'sfv': c.software.version, - 'cmdn': c.name - } - for k in commands[cmd]['del_keys']: - del params[k] - - # making sure that the reference is only created once - if 'reference' in params: - rid = params.pop('reference') - if rid not in refs: - reference = qdb.reference.Reference(rid) - tfp = basename(reference.taxonomy_fp) - sfp = basename(reference.sequence_fp) - refs[rid] = { - 'name': reference.name, - 'taxonomy_fp': tfp, - 'sequence_fp': sfp, - 'tree_fp': basename(reference.tree_fp), - 'version': reference.version - } - params['reference_name'] = refs[rid]['name'] - params['reference_version'] = refs[rid][ - 'version'] - - proc_info['algorithm'] = '%s v%s (%s)' % ( - commands[cmd]['sfwn'], commands[cmd]['sfv'], - commands[cmd]['cmdn']) - proc_info['params'] = params - - info["proc_data_info"].append(proc_info) - infolist.append({ 'metadata_complete': info['metadata_complete'], 'publication_pid': info['publication_pid'], 'ebi_submission_status': info['ebi_submission_status'], 'shared': info['shared'], 'study_abstract': info['study_abstract'], 'pi': info['pi'], - 'status': info['status'], - 'proc_data_info': info['proc_data_info'], + 'status': qdb.study.Study(info['study_id']).status, 'study_tags': info['study_tags'], 'publication_doi': info['publication_doi'], 'study_id': info['study_id'], 'ebi_study_accession': info['ebi_study_accession'], 'study_title': info['study_title'], - 'number_samples_collected': info['number_samples_collected'] + 'number_samples_collected': info['number_samples_collected'], + 'artifact_biom_ids': info['artifact_biom_ids'] }) return infolist diff --git a/qiita_pet/handlers/api_proxy/__init__.py b/qiita_pet/handlers/api_proxy/__init__.py index 68a6956e1..514d4a759 100644 --- a/qiita_pet/handlers/api_proxy/__init__.py +++ b/qiita_pet/handlers/api_proxy/__init__.py @@ -32,7 +32,8 @@ artifact_post_req, artifact_get_req, artifact_status_put_req, artifact_delete_req, artifact_summary_get_request, artifact_get_prep_req, - artifact_summary_post_request, artifact_patch_request) + artifact_summary_post_request, artifact_patch_request, + artifact_get_biom_info) from .ontology import ontology_patch_handler from .processing import ( list_commands_handler_get_req, process_artifact_handler_get_req, @@ -54,7 +55,7 @@ 'artifact_delete_req', 'prep_template_get_req', 'study_delete_req', 'study_prep_get_req', 'sample_template_get_req', 'artifact_graph_get_req', 'artifact_types_get_req', - 'artifact_post_req', + 'artifact_post_req', 'artifact_get_biom_info', 'sample_template_meta_cats_get_req', 'sample_template_samples_get_req', 'prep_template_samples_get_req', 'sample_template_category_get_req', 'new_prep_template_get_req', diff --git a/qiita_pet/handlers/api_proxy/artifact.py b/qiita_pet/handlers/api_proxy/artifact.py index d6e04e6b3..ed0532446 100644 --- a/qiita_pet/handlers/api_proxy/artifact.py +++ b/qiita_pet/handlers/api_proxy/artifact.py @@ -296,6 +296,45 @@ def artifact_get_prep_req(user_id, artifact_ids): return {'status': 'success', 'msg': '', 'data': samples} +@execute_as_transaction +def artifact_get_biom_info(user_id, artifact_ids): + """Returns all artifact info for the given artifact_ids + + Parameters + ---------- + user_id : str + user making the request + artifact_ids : list of int + list of artifact ids + + Returns + ------- + dict of objects + A dictionary containing the artifact information + {'status': status, + 'message': message, + 'data': {artifact_id: {biom_info}} + """ + artifact_info = {} + + for aid in artifact_ids: + artifact = Artifact(aid) + access_error = check_access(artifact.study.id, user_id) + if access_error: + return access_error + + info = artifact.biom_info + if info is not None: + info['timestamp'] = str(info['timestamp']) + else: + # sending an empty string is better than sending None for web + # and testing + info = '' + artifact_info[aid] = info + + return {'status': 'success', 'msg': '', 'data': artifact_info} + + @execute_as_transaction def artifact_post_req(user_id, filepaths, artifact_type, name, prep_template_id, artifact_id=None): diff --git a/qiita_pet/handlers/api_proxy/tests/test_artifact.py b/qiita_pet/handlers/api_proxy/tests/test_artifact.py index 564bdd429..ce5d1a0b0 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_artifact.py +++ b/qiita_pet/handlers/api_proxy/tests/test_artifact.py @@ -20,7 +20,7 @@ from qiita_db.artifact import Artifact from qiita_db.metadata_template.prep_template import PrepTemplate from qiita_db.study import Study -from qiita_db.util import get_mountpoint +from qiita_db.util import get_mountpoint, get_db_files_base_dir from qiita_db.processing_job import ProcessingJob from qiita_db.user import User from qiita_db.software import Command, Parameters, DefaultParameters @@ -29,7 +29,7 @@ artifact_get_req, artifact_status_put_req, artifact_graph_get_req, artifact_delete_req, artifact_types_get_req, artifact_post_req, artifact_summary_get_request, artifact_summary_post_request, - artifact_patch_request, artifact_get_prep_req) + artifact_patch_request, artifact_get_prep_req, artifact_get_biom_info) class TestArtifactAPIReadOnly(TestCase): @@ -428,6 +428,45 @@ def test_artifact_get_prep_req(self): 'message': 'User does not have access to study'} self.assertEqual(obs, exp) + def test_artifact_get_biom_info(self): + bdir = get_db_files_base_dir() + + obs = artifact_get_biom_info('test@foo.bar', [5, 6]) + exp = {'status': 'success', 'msg': '', 'data': { + 5: {'files': [(9, join(bdir, ('processed_data/1_study_1001_closed' + '_reference_otu_table.biom')))], + 'target_subfragment': ['V4'], 'parameters': { + 'reference': 1, 'similarity': 0.97, 'sortmerna_e_value': 1, + 'sortmerna_max_pos': 10000, 'input_data': 2, 'threads': 1, + 'sortmerna_coverage': 0.97}, + 'algorithm': ('Pick closed-reference OTUs | Split libraries ' + 'FASTQ (Defaults with reverse complement ' + 'mapping file barcodes)'), + 'timestamp': '2012-10-02 17:30:00', + 'data_type': '18S', 'name': 'BIOM'}, + 6: {'files': [(12, join(bdir, ( + 'processed_data/1_study_1001_closed_reference_otu_' + 'table_Silva.biom')))], + 'target_subfragment': ['V4'], 'parameters': { + 'reference': 2, 'similarity': 0.97, 'sortmerna_e_value': 1, + 'sortmerna_max_pos': 10000, 'input_data': 2, 'threads': 1, + 'sortmerna_coverage': 0.97}, 'algorithm': ( + 'Pick closed-reference OTUs | Split libraries FASTQ ' + '(Defaults with reverse complement mapping file ' + 'barcodes)'), + 'timestamp': '2012-10-02 17:30:00', + 'data_type': '16S', 'name': 'BIOM'}}} + self.assertEqual(obs, exp) + + obs = artifact_get_biom_info('demo@microbio.me', [4]) + exp = {'status': 'error', + 'message': 'User does not have access to study'} + self.assertEqual(obs, exp) + + obs = artifact_get_biom_info('test@foo.bar', [7]) + exp = {'status': 'success', 'msg': '', 'data': {7: ''}} + self.assertEqual(obs, exp) + def test_artifact_post_req(self): # Create new prep template to attach artifact to pt = npt.assert_warns( diff --git a/qiita_pet/handlers/study_handlers/__init__.py b/qiita_pet/handlers/study_handlers/__init__.py index 0cd6011d3..a635c403c 100644 --- a/qiita_pet/handlers/study_handlers/__init__.py +++ b/qiita_pet/handlers/study_handlers/__init__.py @@ -22,7 +22,7 @@ WorkflowRunHandler, JobAJAX) from .artifact import (ArtifactGraphAJAX, NewArtifactHandler, ArtifactAdminAJAX, ArtifactSummaryAJAX, - ArtifactGetSamples) + ArtifactGetSamples, ArtifactGetBIOMInfo) from .sample_template import SampleTemplateAJAX, SampleAJAX __all__ = ['ListStudiesHandler', 'StudyApprovalList', 'ShareStudyAJAX', @@ -36,4 +36,4 @@ 'DataTypesMenuAJAX', 'StudyFilesAJAX', 'PrepTemplateSummaryAJAX', 'ArtifactSummaryAJAX', 'WorkflowHandler', 'WorkflowRunHandler', 'JobAJAX', 'AutocompleteHandler', 'StudyGetTags', 'StudyTags', - 'ArtifactGetSamples'] + 'ArtifactGetSamples', 'ArtifactGetBIOMInfo'] diff --git a/qiita_pet/handlers/study_handlers/artifact.py b/qiita_pet/handlers/study_handlers/artifact.py index 9501c65fe..4884b2bed 100644 --- a/qiita_pet/handlers/study_handlers/artifact.py +++ b/qiita_pet/handlers/study_handlers/artifact.py @@ -16,7 +16,7 @@ artifact_graph_get_req, artifact_types_get_req, artifact_post_req, artifact_status_put_req, artifact_get_req, artifact_summary_get_request, artifact_summary_post_request, - artifact_get_prep_req) + artifact_get_prep_req, artifact_get_biom_info) from qiita_core.util import execute_as_transaction from qiita_core.qiita_settings import qiita_config @@ -87,6 +87,16 @@ def get(self): self.write(response) +class ArtifactGetBIOMInfo(BaseHandler): + @authenticated + def get(self): + aids = map(int, self.request.arguments.get('ids[]', [])) + + response = artifact_get_biom_info(self.current_user.id, aids) + + self.write(response) + + class ArtifactAdminAJAX(BaseHandler): @authenticated def get(self): diff --git a/qiita_pet/handlers/study_handlers/tests/test_artifact.py b/qiita_pet/handlers/study_handlers/tests/test_artifact.py index 880699d04..dc7a1e874 100644 --- a/qiita_pet/handlers/study_handlers/tests/test_artifact.py +++ b/qiita_pet/handlers/study_handlers/tests/test_artifact.py @@ -19,7 +19,7 @@ from qiita_pet.test.tornado_test_base import TestHandlerBase from qiita_db.artifact import Artifact from qiita_db.study import Study -from qiita_db.util import get_mountpoint +from qiita_db.util import get_mountpoint, get_db_files_base_dir from qiita_db.metadata_template.prep_template import PrepTemplate from qiita_db.exceptions import QiitaDBWarning @@ -156,6 +156,33 @@ def test_get(self): self.assertEqual(loads(response.body), exp) +class ArtifactGetBIOMInfoTest(TestHandlerBase): + def test_get(self): + bdir = get_db_files_base_dir() + + response = self.get('/artifact/info/', {'ids[]': [6, 7]}) + self.assertEqual(response.code, 200) + exp = ( + {'status': 'success', 'msg': '', 'data': { + '7': '', + '6': { + 'files': [[12, join(bdir, ( + 'processed_data/1_study_1001_closed_reference_' + 'otu_table_Silva.biom'))]], + 'target_subfragment': ['V4'], + 'algorithm': ('Pick closed-reference OTUs | Split ' + 'libraries FASTQ (Defaults with reverse ' + 'complement mapping file barcodes)'), + 'data_type': '16S', 'timestamp': '2012-10-02 17:30:00', + 'parameters': {'reference': 2, 'similarity': 0.97, + 'sortmerna_e_value': 1, + 'sortmerna_max_pos': 10000, + 'input_data': 2, 'threads': 1, + 'sortmerna_coverage': 0.97}, + 'name': 'BIOM'}}}) + self.assertEqual(loads(response.body), exp) + + class ArtifactAdminAJAXTestsReadOnly(TestHandlerBase): def test_get_admin(self): response = self.get('/admin/artifact/', diff --git a/qiita_pet/handlers/study_handlers/tests/test_listing_handlers.py b/qiita_pet/handlers/study_handlers/tests/test_listing_handlers.py index e4de44bb5..13ab0e0f2 100644 --- a/qiita_pet/handlers/study_handlers/tests/test_listing_handlers.py +++ b/qiita_pet/handlers/study_handlers/tests/test_listing_handlers.py @@ -24,25 +24,6 @@ GPARAMS = {'similarity': 0.97, 'reference_name': 'Greengenes', 'sortmerna_e_value': 1, 'sortmerna_max_pos': 10000, 'threads': 1, 'sortmerna_coverage': 0.97, 'reference_version': u'13_8'} -PROC_DATA_INFO = [ - {'data_type': u'18S', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 4, 'processed_date': '2012-10-02 17:30:00', 'params': GPARAMS, - 'name': 'BIOM'}, - {'data_type': '18S', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 5, 'processed_date': '2012-10-02 17:30:00', 'params': GPARAMS, - 'name': 'BIOM'}, - {'data_type': '16S', - 'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)', - 'pid': 6, 'processed_date': '2012-10-02 17:30:00', - 'params': {'similarity': 0.97, 'reference_name': u'Silva', - 'sortmerna_e_value': 1, 'sortmerna_max_pos': 10000, - 'threads': 1, 'sortmerna_coverage': 0.97, - 'reference_version': 'test'}, - 'name': 'BIOM'}, - {'processed_date': '2012-10-02 17:30:00', 'pid': 7, 'data_type': '16S', - 'name': 'BIOM'}] class TestHelpers(TestHandlerBase): @@ -74,8 +55,8 @@ def setUp(self): 'publication_doi': ['10.100/123456', '10.100/7891011'], 'publication_pid': ['123456', '7891011'], 'pi': ('PI_dude@foo.bar', 'PIDude'), + 'artifact_biom_ids': [4, 5, 6, 7], 'study_tags': None, - 'proc_data_info': PROC_DATA_INFO } self.exp = [self.single_exp] @@ -142,15 +123,14 @@ def test_build_study_info_empty_study(self): 'publication_pid': [], 'pi': ('PI_dude@foo.bar', 'PIDude'), 'status': 'sandbox', - 'proc_data_info': [], 'publication_doi': [], 'study_abstract': 'abstract', 'study_id': 2, 'ebi_study_accession': None, 'study_title': 'My study', 'study_tags': None, + 'artifact_biom_ids': None, 'number_samples_collected': 0}) - self.assertItemsEqual(obs, self.exp) # Now testing that admin also sees this study @@ -277,7 +257,6 @@ def setUp(self): ' (submitted)'), 'study_title': ('Identification of the Microbiomes for ' 'Cannabis Soils'), - 'proc_data_info': PROC_DATA_INFO, 'metadata_complete': True, 'ebi_submission_status': 'submitted', 'study_id': 1, @@ -305,6 +284,7 @@ def setUp(self): 'Future studies will attempt to analyze the soils and ' 'rhizospheres from the same location at different time ' 'points in the plant lifecycle.'), + 'artifact_biom_ids': [4, 5, 6, 7], 'number_samples_collected': 27, 'study_tags': None}], 'sEcho': 1021, diff --git a/qiita_pet/static/css/style.css b/qiita_pet/static/css/style.css index db6cc183d..efb2e4b48 100644 --- a/qiita_pet/static/css/style.css +++ b/qiita_pet/static/css/style.css @@ -18,10 +18,18 @@ td.more-info-processing-jobs{ cursor: pointer; - background: url('../img//details_open.png') no-repeat center center; + background: url('../img/details_open.png') no-repeat center center; } -tr.shown td.more-info-processing-jobs { - background: url('../img//details_close.png') no-repeat center center; +tr.shown td.more-info-processing-jobs{ + background: url('../img/details_close.png') no-repeat center center; +} + +td.details-control{ + cursor: pointer; + background: url('../img/details_open.png') no-repeat center center; +} +tr.shown td.details-control{ + background: url('../img/details_close.png') no-repeat center center; } .blinking-message { diff --git a/qiita_pet/templates/list_studies.html b/qiita_pet/templates/list_studies.html index d3a4de4ac..ca27433ca 100644 --- a/qiita_pet/templates/list_studies.html +++ b/qiita_pet/templates/list_studies.html @@ -35,9 +35,9 @@ var row_data = $('#'+name).dataTable().fnGetData(row); var aids = [] - for(var i=0;i' + - 'IDNameData typeProcessed Date' + - 'AlgorithmParameters'; - for(i=0;i'; + proc_data_table += '' + pid + ''; + proc_data_table += '' + info.name + ''; + proc_data_table += '' + info.data_type + ''; + proc_data_table += '' + info.target_subfragment.join(', ') + ''; + proc_data_table += '' + info.timestamp + ''; + proc_data_table += '' + info.algorithm + ''; + var params = ''; - var p = proc_data.params; - for (var key in p) { - params += '' + key + ': ' + p[key] + '
'; + for (var key in info.parameters) { + params += '' + key + ': ' + info.parameters[key] + '
'; + } + proc_data_table += '' + params + ''; + + var files = ''; + for (var key in info.files) { + var vals = info.files[key] + var name = vals[1].split('/'); + files += 'FID: ' + vals[0]; + files += '
'; + files += 'Name: ' + name[name.length-1]; } - proc_data_table += '' + - proc_data.pid + '' + proc_data.name + '' + proc_data.data_type + '' + - proc_data.processed_date + '' + - proc_data.algorithm + '' + params + ''; + proc_data_table += '' + files + ''; + + proc_data_table += ''; } - proc_data_table += ''; - return proc_data_table; + }); + + proc_data_table += ''; + return proc_data_table; } $('#user-studies-table').dataTable({ "lengthMenu": [[5, 10, 50, -1], [5, 10, 50, "All"]], "deferRender": true, "columns": [ - {"className": 'details-control', "orderable": false, "data": null, "defaultContent": ''}, + {"className": 'details-control', "orderable": false, "data": null, "defaultContent": ''}, { "orderable": false}, { "data": "study_title" }, { "data": "study_abstract" }, @@ -132,10 +157,11 @@ {"targets": [ 3 ], "visible": false}, // render the study checkbox cell {"render": function ( data, type, row, meta ) { - if(row.proc_data_info.length > 0) { - return ""; - } - else { return '

No Processed Data

'; } + return row.artifact_biom_ids.length + // if(row.proc_data_info.length > 0) { + // return ""; + // } + // else { return '

No Processed Data

'; } }, targets: [1]}, // render the title cell {"render": function ( data, type, row, meta ) { @@ -196,10 +222,10 @@ {"targets": [ 3 ], "visible": false}, // render the study checkbox cell {"render": function ( data, type, row, meta ) { - if(row.proc_data_info.length > 0) { - return ""; - } - else { return '

No Processed Data

'; } + // if(row.proc_data_info.length > 0) { + // return ""; + // } + // else { return '

No Processed Data

'; } }, targets: [1]}, // render the title cell {"render": function ( data, type, row, meta ) { @@ -239,15 +265,13 @@ var table = $('#studies-table').DataTable(); var tr = $(this).closest('tr'); var row = table.row( tr ); - if ( row.child.isShown() ) { // This row is already open - close it row.child.hide(); tr.removeClass('shown'); - } - else { + } else { // Open this row - row.child( format('studies-table', row.data(), row.index()) ).show(); + row.child( format_biom_rows('studies-table', row.data(), row.index()) ).show(); tr.addClass('shown'); } }); @@ -255,16 +279,23 @@ var table = $('#user-studies-table').DataTable(); var tr = $(this).closest('tr'); var row = table.row( tr ); - if ( row.child.isShown() ) { // This row is already open - close it row.child.hide(); tr.removeClass('shown'); - } - else { + } else { // Open this row - row.child( format('user-studies-table', row.data(), row.index()) ).show(); + // modified from: https://jsfiddle.net/8rejaL88/2/ tr.addClass('shown'); + row.child('

', 'no-padding' ).show(); + $.get('/artifact/info/', {ids: row.data().artifact_biom_ids}) + .done(function ( data ) { + if (data['status']=='success') { + $('td', row.child()).html(format_biom_rows('studies-table', data.data, row.index())).show(); + } else { + bootstrapAlert('ERROR: ' + data['msg'], "danger", 10000); + } + }); } }); @@ -376,7 +407,7 @@

Your Studies (includes shared with you)

- + @@ -392,19 +423,19 @@

Your Studies (includes shared with you)

ExpandFor analysis Add to analysis Title Abstract

Other Studies

- - - - - - - - - - - - - + + + + + + + + + + + + +
ExpandAdd to analysisTitleAbstractStudy IDSamplesPrincipal InvestigatorPublicationsEBI
For analysisAdd to analysisTitleAbstractStudy IDSamplesPrincipal InvestigatorPublicationsEBI
diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index ca3184691..9a8a6d3fe 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -30,8 +30,8 @@ ListCommandsHandler, ListOptionsHandler, PrepTemplateSummaryAJAX, PrepTemplateAJAX, NewArtifactHandler, SampleAJAX, StudyDeleteAjax, ArtifactAdminAJAX, NewPrepTemplateAjax, DataTypesMenuAJAX, StudyFilesAJAX, - ArtifactGetSamples, WorkflowHandler, WorkflowRunHandler, JobAJAX, - AutocompleteHandler) + ArtifactGetSamples, ArtifactGetBIOMInfo, WorkflowHandler, + WorkflowRunHandler, JobAJAX, AutocompleteHandler) from qiita_pet.handlers.artifact_handlers import ( ArtifactSummaryAJAX, ArtifactAJAX, ArtifactSummaryHandler, ProcessArtifactHandler) @@ -106,12 +106,13 @@ def __init__(self): (r"/analysis/description/(.*)/graph/", AnalysisGraphHandler), (r"/analysis/description/(.*)/jobs/", AnalysisJobsHandler), (r"/analysis/description/(.*)/", AnalysisDescriptionHandler), + (r"/artifact/samples/", ArtifactGetSamples), + (r"/artifact/info/", ArtifactGetBIOMInfo), (r"/moi-ws/", MOIMessageHandler), (r"/consumer/", MessageHandler), (r"/admin/error/", LogEntryViewerHandler), (r"/admin/approval/", StudyApprovalList), (r"/admin/artifact/", ArtifactAdminAJAX), - (r"/artifact/samples/", ArtifactGetSamples), (r"/ebi_submission/(.*)", EBISubmitHandler), # Study handlers (r"/study/create/", StudyEditHandler), From c97c34a82fe08682bb176e0cfc56fe37f6d54237 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sun, 18 Jun 2017 06:28:59 -0600 Subject: [PATCH 03/18] adding code to both list --- qiita_pet/templates/list_studies.html | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/qiita_pet/templates/list_studies.html b/qiita_pet/templates/list_studies.html index ca27433ca..eb6394823 100644 --- a/qiita_pet/templates/list_studies.html +++ b/qiita_pet/templates/list_studies.html @@ -87,7 +87,7 @@ $("#search-waiting").hide(); moi.init(null, window.location.host + '{% raw qiita_config.portal_dir %}/study/list/socket/', function(){}, error, error); moi.add_callback('sel', show_alert); - function format_biom_rows(name, data, row) { + function format_biom_rows(data, row) { var proc_data_table = ''; proc_data_table += ''; proc_data_table += ''; @@ -265,14 +265,24 @@ var table = $('#studies-table').DataTable(); var tr = $(this).closest('tr'); var row = table.row( tr ); + if ( row.child.isShown() ) { // This row is already open - close it row.child.hide(); tr.removeClass('shown'); } else { // Open this row - row.child( format_biom_rows('studies-table', row.data(), row.index()) ).show(); + // modified from: https://jsfiddle.net/8rejaL88/2/ tr.addClass('shown'); + row.child('

', 'no-padding' ).show(); + $.get('/artifact/info/', {ids: row.data().artifact_biom_ids}) + .done(function ( data ) { + if (data['status']=='success') { + $('td', row.child()).html(format_biom_rows(data.data, row.index())).show(); + } else { + bootstrapAlert('ERROR: ' + data['msg'], "danger", 10000); + } + }); } }); $('#user-studies-table tbody').on('click', 'td.details-control', function () { @@ -291,7 +301,7 @@ $.get('/artifact/info/', {ids: row.data().artifact_biom_ids}) .done(function ( data ) { if (data['status']=='success') { - $('td', row.child()).html(format_biom_rows('studies-table', data.data, row.index())).show(); + $('td', row.child()).html(format_biom_rows(data.data, row.index())).show(); } else { bootstrapAlert('ERROR: ' + data['msg'], "danger", 10000); } From 82d9798b41940b9e2a302ff1aa3ac057051c35ef Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sun, 18 Jun 2017 07:50:02 -0600 Subject: [PATCH 04/18] fix js length --- qiita_pet/templates/list_studies.html | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/qiita_pet/templates/list_studies.html b/qiita_pet/templates/list_studies.html index eb6394823..ca85a96d3 100644 --- a/qiita_pet/templates/list_studies.html +++ b/qiita_pet/templates/list_studies.html @@ -157,7 +157,11 @@ {"targets": [ 3 ], "visible": false}, // render the study checkbox cell {"render": function ( data, type, row, meta ) { - return row.artifact_biom_ids.length + var len = 0; + if (row.artifact_biom_ids != undefined) { + len = row.artifact_biom_ids.length; + } + return len // if(row.proc_data_info.length > 0) { // return ""; // } @@ -207,7 +211,7 @@ "sDom": '<"top">rti<"bottom"p><"clear">', "bLengthChange": false, "columns": [ - {"className": 'details-control', "orderable": false, "data": null, "defaultContent": ''}, + {"className": 'details-control', "orderable": false, "data": null, "defaultContent": ''}, { "orderable": false}, { "data": "study_title" }, { "data": "study_abstract" }, @@ -222,8 +226,13 @@ {"targets": [ 3 ], "visible": false}, // render the study checkbox cell {"render": function ( data, type, row, meta ) { + var len = 0; + if (row.artifact_biom_ids != undefined) { + len = row.artifact_biom_ids.length; + } + return len // if(row.proc_data_info.length > 0) { - // return ""; + // return ""; // } // else { return '

No Processed Data

'; } }, targets: [1]}, From ddd724af4c2f45b535cadb063d899b2550dd2710 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 20 Jun 2017 11:38:08 -0600 Subject: [PATCH 05/18] improving sql and display --- qiita_db/artifact.py | 68 --------- qiita_db/test/test_artifact.py | 31 ---- qiita_db/test/test_util.py | 26 ++++ qiita_db/util.py | 138 ++++++++++++++++++ qiita_pet/handlers/api_proxy/artifact.py | 18 +-- .../handlers/api_proxy/tests/test_artifact.py | 57 +++----- qiita_pet/static/css/style.css | 41 ------ qiita_pet/static/js/qiita.js | 7 +- qiita_pet/templates/list_studies.html | 17 +-- qiita_pet/templates/sitebase.html | 2 +- 10 files changed, 198 insertions(+), 207 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 9a3d0ac03..3113a4bd7 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -1181,74 +1181,6 @@ def analysis(self): res = qdb.sql_connection.TRN.execute_fetchindex() return qdb.analysis.Analysis(res[0][0]) if res else None - @property - def biom_info(self): - """Returns processing information about the bioms in the artifact - - Returns - ------- - dict or None - The info of the bioms if artifact_type is BIOM or None if not. - """ - if self.artifact_type != 'BIOM': - return None - files = [(fid, fp)for fid, fp, fpt in self.filepaths if fpt == 'biom'] - if not files: - return None - - parameters = {} - with qdb.sql_connection.TRN: - # getting target_subfragment - sql = """ - SELECT DISTINCT target_subfragment - FROM qiita.prep_%s - WHERE EXISTS (SELECT 1 - FROM information_schema.columns - WHERE table_name='prep_%s' - AND column_name='target_subfragment')""" - pt_ids = [[pt.id, pt.id] for pt in self.prep_templates] - qdb.sql_connection.TRN.add(sql, pt_ids, many=True) - target_subfragment = qdb.sql_connection.TRN.execute_fetchflatten() - - # getting processing_parameters - pp = self.processing_parameters - if pp is not None: - parameters = pp.values - parents = self.parents - if bool(parents): - # [0] an artifact can only have one processing parent - parent = parents[0] - ppp = parent.processing_parameters.command.name - # obtaining all processing parameters so we can then - # match, the parents processing params - sql_params = """ - SELECT parameter_set_name, array_agg(ps) AS param_set - FROM qiita.default_parameter_set, - json_each_text(parameter_set) ps - GROUP BY parameter_set_name""" - qdb.sql_connection.TRN.add(sql_params) - params = {pname: eval(params) for pname, params - in qdb.sql_connection.TRN.execute_fetchindex()} - tpppv = {'(%s,%s)' % (k, v) - for k, v in viewitems( - parent.processing_parameters.values)} - pppv = sorted([ - [k, len(tpppv & v)] for k, v in viewitems(params)], - key=lambda x: x[1])[-1][0] - else: - ppp = '' - pppv = '' - - algorithm = '%s | %s (%s)' % ( - pp.command.name, ppp, pppv) - else: - algorithm = 'N/A' - - return {'target_subfragment': target_subfragment, 'name': self.name, - 'data_type': self.data_type, 'timestamp': self.timestamp, - 'parameters': parameters, 'algorithm': algorithm, - 'files': files} - def jobs(self, cmd=None, status=None): """Jobs that used this artifact as input diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 6d3f29767..0f51c498c 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -407,37 +407,6 @@ def test_analysis(self): qdb.analysis.Analysis(1)) self.assertIsNone(qdb.artifact.Artifact(1).analysis) - def test_biom_info(self): - # testing not biom - self.assertIsNone(qdb.artifact.Artifact(1).biom_info) - self.assertIsNone(qdb.artifact.Artifact(2).biom_info) - # test no biom // empty analysis - self.assertIsNone(qdb.artifact.Artifact(7).biom_info) - # regular processed file - bdir = qdb.util.get_db_files_base_dir() - obs = qdb.artifact.Artifact(4).biom_info - exp = { - 'files': [(9, join(bdir, 'processed_data/1_study_1001_closed' - '_reference_otu_table.biom'))], - 'target_subfragment': ['V4'], 'parameters': { - 'reference': 1, 'similarity': 0.97, 'sortmerna_e_value': 1, - 'sortmerna_max_pos': 10000, 'input_data': 2, 'threads': 1, - 'sortmerna_coverage': 0.97}, - 'algorithm': ( - 'Pick closed-reference OTUs | Split libraries FASTQ ' - '(Defaults with reverse complement mapping file barcodes)'), - 'timestamp': datetime(2012, 10, 2, 17, 30), - 'data_type': '18S', 'name': 'BIOM'} - self.assertEqual(obs, exp) - # analysis - obs = qdb.artifact.Artifact(8).biom_info - exp = { - 'files': [(15, join(bdir, 'analysis/1_analysis_18S.biom'))], - 'target_subfragment': [], 'parameters': {}, 'algorithm': 'N/A', - 'timestamp': datetime(2017, 6, 16, 15, 28, 28, 943120), - 'data_type': '18S', 'name': 'noname'} - self.assertEqual(obs, exp) - def test_jobs(self): obs = qdb.artifact.Artifact(1).jobs() exp = [ diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 288689442..27bc4e775 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -846,6 +846,32 @@ def test_generate_study_list(self): obs_info = qdb.util.generate_study_list([1, 2, 3, 4], False) self.assertEqual(obs_info, exp_info) + def test_get_artifacts_bioms_information(self): + # we are gonna test that it ignores 1 and 2 cause they are not biom, + # 4 has all information and 7 and 8 don't + obs = qdb.util.get_artifacts_bioms_information([1, 2, 4, 7, 8]) + # not testing timestamp + for i in range(len(obs)): + del obs[i]['timestamp'] + exp = [ + {'files': ['1_study_1001_closed_reference_otu_table.biom'], + 'target_subfragment': ['V4'], + 'algorithm': 'Pick closed-reference OTUs, QIIMEv1.9.1 | Defaults', + 'artifact_id': 4, 'data_type': '18S', + 'parameters': {'reference': 1, 'similarity': 0.97, + 'sortmerna_e_value': 1, 'sortmerna_max_pos': 10000, + 'input_data': 2, 'threads': 1, + 'sortmerna_coverage': 0.97}, + 'name': 'BIOM'}, { + 'files': [], + 'target_subfragment': ['V4'], + 'algorithm': '', 'artifact_id': 7, 'data_type': '16S', + 'parameters': {}, 'name': 'BIOM'}, + {'files': ['1_analysis_18S.biom'], 'target_subfragment': [], + 'algorithm': '', 'artifact_id': 8, 'data_type': '18S', + 'parameters': {}, 'name': 'noname'}] + self.assertEqual(obs, exp) + if __name__ == '__main__': main() diff --git a/qiita_db/util.py b/qiita_db/util.py index bb1d974fb..1b29c102b 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -1317,3 +1317,141 @@ def generate_study_list(study_ids, public_only=False): }) return infolist + + +def get_artifacts_bioms_information(artifact_ids): + """Returns processing information about the bioms in the artifact + + Returns + ------- + dict or None + The info of the bioms if artifact_type is BIOM or None if not. + """ + sql = """ + SELECT a.artifact_id, a.name, a.command_id, a.generated_timestamp, + array_agg(a.command_parameters), dt.data_type, parent_id, + array_agg(parent_info.command_parameters), + array_agg(filepaths.filepath), array_agg(pt.prep_id) + FROM qiita.artifact a + JOIN qiita.artifact_type at ON ( + a.artifact_type_id = at .artifact_type_id + AND artifact_type = 'BIOM') + LEFT JOIN qiita.parent_artifact pa ON ( + a.artifact_id = pa.artifact_id) + LEFT OUTER JOIN LATERAL ( + SELECT command_parameters FROM qiita.artifact ap + WHERE ap.artifact_id = pa.parent_id) parent_info ON true + LEFT OUTER JOIN LATERAL ( + SELECT filepath + FROM qiita.artifact_filepath af + JOIN qiita.filepath USING (filepath_id) + WHERE af.artifact_id = a.artifact_id) filepaths ON true + LEFT OUTER JOIN LATERAL ( + SELECT data_type + FROM qiita.data_type + WHERE data_type_id = a.data_type_id) dt ON true + LEFT OUTER JOIN LATERAL ( + SELECT CASE WHEN ( + SELECT true + FROM information_schema.columns + WHERE table_name = 'prep_' || CAST( + prep_template_id AS TEXT) + AND column_name='target_subfragment') + THEN prep_template_id + ELSE null END AS prep_id + FROM qiita.prep_template pt + WHERE pt.artifact_id IN ( + SELECT * FROM qiita.find_artifact_roots(a.artifact_id))) + pt ON true + WHERE a.artifact_id IN %s + GROUP BY a.artifact_id, a.name, a.command_id, + a.generated_timestamp, dt.data_type, parent_id + ORDER BY command_id, artifact_id + """ + + sql_params = """ + SELECT parameter_set_name, array_agg(parameter_set) AS param_set + FROM qiita.default_parameter_set + GROUP BY parameter_set_name""" + + sql_ts = """SELECT DISTINCT target_subfragment FROM qiita.prep_%s""" + + with qdb.sql_connection.TRN: + results = [] + commands = {} + # obtaining all existing parameters, note that + # they are not that many (~40) and we don't expect + # to have a huge growth in the near future + qdb.sql_connection.TRN.add(sql_params) + params = {name: set(params[0].iteritems()) for name, params in + qdb.sql_connection.TRN.execute_fetchindex()} + + # now let's get the actual artifacts + qdb.sql_connection.TRN.add(sql, [tuple(artifact_ids)]) + for row in qdb.sql_connection.TRN.execute_fetchindex(): + aid, name, cid, gt, aparams, dt, pid, pparams, filepaths, \ + target = row + # cleaning fields: + # - [0] due to the array_agg + pparams = pparams[0] + if pparams is not None: + # [-1] taking the last cause it's sorted by + # the number of overlapping parameters + # [0] then taking the first element that is + # the name of the parameter set + pparams = sorted([ + [k, len(v & set(pparams.iteritems()))] + for k, v in viewitems(params)], + key=lambda x: x[1])[-1][0] + else: + pparams = 'N/A' + # - [0] due to the array_agg + aparams = aparams[0] + if aparams is None: + aparams = {} + # - ignoring empty filepaths + if filepaths == [None]: + filepaths = [] + # - ignoring empty target + if target == [None]: + target = [] + + algorithm = '' + if cid is not None: + if cid not in commands: + c = qdb.software.Command(cid) + s = c.software + commands[cid] = '%s, %sv%s' % ( + c.name, s.name, s.version) + + algorithm = '%s | %s' % (commands[cid], pparams) + + results.append({ + 'artifact_id': aid, + 'target_subfragment': target, + 'name': name, + 'data_type': dt, + 'timestamp': str(gt), + 'parameters': aparams, + 'algorithm': algorithm, + 'files': filepaths}) + + # let's get the values for target_subfragment from the + # prep_template, note that we have to do it in a separate sql + # doing crosstab is really difficult and in another loop cause we + # need to loop over all execute_fetchindex before doing another + # query + ts = {} + for i, r in enumerate(results): + ats = [] + for pid in r['target_subfragment']: + if pid not in ts: + qdb.sql_connection.TRN.add(sql_ts, [pid]) + ts[pid] = qdb.sql_connection.TRN.execute_fetchflatten() + ats.extend(ts[pid]) + + # set to remove any duplicates, then list so JSON can serialize + # and play nice with web + results[i]['target_subfragment'] = list(set(ats)) + + return results diff --git a/qiita_pet/handlers/api_proxy/artifact.py b/qiita_pet/handlers/api_proxy/artifact.py index ed0532446..0a668b0b9 100644 --- a/qiita_pet/handlers/api_proxy/artifact.py +++ b/qiita_pet/handlers/api_proxy/artifact.py @@ -21,7 +21,8 @@ from qiita_db.artifact import Artifact from qiita_db.user import User from qiita_db.metadata_template.prep_template import PrepTemplate -from qiita_db.util import get_mountpoint, get_visibilities +from qiita_db.util import ( + get_mountpoint, get_visibilities, get_artifacts_bioms_information) from qiita_db.software import Command, Parameters from qiita_db.processing_job import ProcessingJob @@ -317,20 +318,7 @@ def artifact_get_biom_info(user_id, artifact_ids): """ artifact_info = {} - for aid in artifact_ids: - artifact = Artifact(aid) - access_error = check_access(artifact.study.id, user_id) - if access_error: - return access_error - - info = artifact.biom_info - if info is not None: - info['timestamp'] = str(info['timestamp']) - else: - # sending an empty string is better than sending None for web - # and testing - info = '' - artifact_info[aid] = info + artifact_info = get_artifacts_bioms_information(artifact_ids) return {'status': 'success', 'msg': '', 'data': artifact_info} diff --git a/qiita_pet/handlers/api_proxy/tests/test_artifact.py b/qiita_pet/handlers/api_proxy/tests/test_artifact.py index ce5d1a0b0..78d944902 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_artifact.py +++ b/qiita_pet/handlers/api_proxy/tests/test_artifact.py @@ -429,42 +429,33 @@ def test_artifact_get_prep_req(self): self.assertEqual(obs, exp) def test_artifact_get_biom_info(self): - bdir = get_db_files_base_dir() - - obs = artifact_get_biom_info('test@foo.bar', [5, 6]) - exp = {'status': 'success', 'msg': '', 'data': { - 5: {'files': [(9, join(bdir, ('processed_data/1_study_1001_closed' - '_reference_otu_table.biom')))], - 'target_subfragment': ['V4'], 'parameters': { + obs = artifact_get_biom_info('test@foo.bar', [5, 6, 7]) + exp = { + 'status': 'success', 'msg': '', 'data': [ + {'files': ['1_study_1001_closed_reference_otu_table.biom'], + 'target_subfragment': ['V4'], + 'algorithm': ('Pick closed-reference OTUs, QIIMEv1.9.1' + ' | Defaults'), 'artifact_id': 5, + 'data_type': '18S', 'timestamp': '2012-10-02 17:30:00', + 'parameters': { 'reference': 1, 'similarity': 0.97, 'sortmerna_e_value': 1, 'sortmerna_max_pos': 10000, 'input_data': 2, 'threads': 1, 'sortmerna_coverage': 0.97}, - 'algorithm': ('Pick closed-reference OTUs | Split libraries ' - 'FASTQ (Defaults with reverse complement ' - 'mapping file barcodes)'), - 'timestamp': '2012-10-02 17:30:00', - 'data_type': '18S', 'name': 'BIOM'}, - 6: {'files': [(12, join(bdir, ( - 'processed_data/1_study_1001_closed_reference_otu_' - 'table_Silva.biom')))], - 'target_subfragment': ['V4'], 'parameters': { - 'reference': 2, 'similarity': 0.97, 'sortmerna_e_value': 1, - 'sortmerna_max_pos': 10000, 'input_data': 2, 'threads': 1, - 'sortmerna_coverage': 0.97}, 'algorithm': ( - 'Pick closed-reference OTUs | Split libraries FASTQ ' - '(Defaults with reverse complement mapping file ' - 'barcodes)'), - 'timestamp': '2012-10-02 17:30:00', - 'data_type': '16S', 'name': 'BIOM'}}} - self.assertEqual(obs, exp) - - obs = artifact_get_biom_info('demo@microbio.me', [4]) - exp = {'status': 'error', - 'message': 'User does not have access to study'} - self.assertEqual(obs, exp) - - obs = artifact_get_biom_info('test@foo.bar', [7]) - exp = {'status': 'success', 'msg': '', 'data': {7: ''}} + 'name': 'BIOM'}, + {'files': ['1_study_1001_closed_reference_otu_table_' + 'Silva.biom'], 'target_subfragment': ['V4'], + 'algorithm': ('Pick closed-reference OTUs, QIIMEv1.9.1' + ' | Defaults'), 'artifact_id': 6, + 'data_type': '16S', 'timestamp': '2012-10-02 17:30:00', + 'parameters': {'reference': 2, 'similarity': 0.97, + 'sortmerna_e_value': 1, + 'sortmerna_max_pos': 10000, 'input_data': 2, + 'threads': 1, 'sortmerna_coverage': 0.97}, + 'name': 'BIOM'}, { + 'files': [], 'target_subfragment': ['V4'], 'algorithm': '', + 'artifact_id': 7, 'data_type': '16S', + 'timestamp': '2012-10-02 17:30:00', 'parameters': {}, + 'name': 'BIOM'}]} self.assertEqual(obs, exp) def test_artifact_post_req(self): diff --git a/qiita_pet/static/css/style.css b/qiita_pet/static/css/style.css index efb2e4b48..a39f550d9 100644 --- a/qiita_pet/static/css/style.css +++ b/qiita_pet/static/css/style.css @@ -51,7 +51,6 @@ tr.shown td.details-control{ display: block; } - .info-menu { word-wrap: break-word; white-space: normal; @@ -84,43 +83,3 @@ tr.shown td.details-control{ height:300px; border: 1px solid #ccc; } - -/* Make the navbar collapse at medium size, not small. -Fixes really ugly middle ground where menu stacking occurs -adapted from http://stackoverflow.com/a/23298184 */ -@media (max-width: 991px) { - .navbar-header { - float: none; - } - .navbar-toggle { - display: block; - } - .navbar-collapse { - border-top: 1px solid transparent; - box-shadow: inset 0 1px 0 rgba(255,255,255,0.1); - } - .navbar-collapse.collapse { - display: none!important; - } - .navbar-nav { - float: none!important; - margin: 7.5px -15px; - } - .navbar-nav>li { - float: none; - } - .navbar-nav>li>a { - padding-top: 10px; - padding-bottom: 10px; - } - .navbar-text { - float: none; - margin: 15px 0; - } - /* since 3.1.0 */ .navbar-collapse.collapse.in { - display: block!important; - } - .collapsing { - overflow: hidden!important; - } -} diff --git a/qiita_pet/static/js/qiita.js b/qiita_pet/static/js/qiita.js index 10b0d1657..77ff78d11 100644 --- a/qiita_pet/static/js/qiita.js +++ b/qiita_pet/static/js/qiita.js @@ -11,9 +11,6 @@ var timeoutHandleForBoostrapAlert = null; function bootstrapAlert(message, severity, timeout){ - // Clear the previous alert - so they don't keep stacking on top of each other - $('#bootstrap-alert').alert('close'); - // make timeout an optional parameter timeout = timeout || -1; @@ -32,14 +29,14 @@ function bootstrapAlert(message, severity, timeout){ alertDiv.append('

Need help? Send us an email.

'); } - $('#qiita-main').prepend(alertDiv); + $('#template-content').prepend(alertDiv); if(timeout > 0) { if (timeoutHandleForBoostrapAlert != null) { window.clearTimeout(timeoutHandleForBoostrapAlert); } timeoutHandleForBoostrapAlert = window.setTimeout(function() { - $('#alert-message').alert('close'); + $('#alert-message').remove(); timeoutHandleForBoostrapAlert = null; }, timeout); } diff --git a/qiita_pet/templates/list_studies.html b/qiita_pet/templates/list_studies.html index ca85a96d3..51fa695ef 100644 --- a/qiita_pet/templates/list_studies.html +++ b/qiita_pet/templates/list_studies.html @@ -101,11 +101,11 @@ proc_data_table += ''; proc_data_table += ''; - $.each(data, function (pid, info) { + $.each(data, function (idx, info) { if (typeof info !== 'string' && !(info instanceof String)) { proc_data_table += ''; - proc_data_table += ''; - proc_data_table += ''; + proc_data_table += ''; + proc_data_table += ''; proc_data_table += ''; proc_data_table += ''; proc_data_table += ''; @@ -117,16 +117,7 @@ params += '' + key + ': ' + info.parameters[key] + '
'; } proc_data_table += ''; - - var files = ''; - for (var key in info.files) { - var vals = info.files[key] - var name = vals[1].split('/'); - files += 'FID: ' + vals[0]; - files += '
'; - files += 'Name: ' + name[name.length-1]; - } - proc_data_table += ''; + proc_data_table += ''; proc_data_table += ''; } diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 17c7c819b..31409d7f8 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -362,7 +362,7 @@
-
Files
' + pid + '' + info.artifact_id + '' + info.name + '' + info.data_type + '' + info.target_subfragment.join(', ') + '' + params + '' + files + '' + info.files.join(', ') + '
'; proc_data_table += ''; proc_data_table += ''; - proc_data_table += ''; proc_data_table += ''; proc_data_table += ''; - proc_data_table += ''; - proc_data_table += ''; - proc_data_table += ''; + proc_data_table += ''; proc_data_table += ''; proc_data_table += ''; proc_data_table += ''; @@ -105,11 +102,8 @@ if (typeof info !== 'string' && !(info instanceof String)) { proc_data_table += ''; proc_data_table += ''; - proc_data_table += ''; - proc_data_table += ''; - proc_data_table += ''; - proc_data_table += ''; - proc_data_table += ''; + proc_data_table += ''; + proc_data_table += ''; proc_data_table += ''; var params = ''; @@ -117,7 +111,7 @@ params += '' + key + ': ' + info.parameters[key] + '
'; } proc_data_table += ''; - proc_data_table += ''; + proc_data_table += ''; proc_data_table += ''; } diff --git a/scripts/qiita b/scripts/qiita index 65153b0ba..bc34508e7 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -464,8 +464,8 @@ def start(port, master): # Set a PeriodicCallback for cleaning up the threads every 10 seconds # To understand why this is working as expected, check the multiprocessing # documentation https://docs.python.org/2/library/multiprocessing.html - # 600000 == 10 min - PeriodicCallback(lambda: active_children(), 600000).start() + # 1200000 == 20 min + PeriodicCallback(lambda: active_children(), 1200000).start() ioloop.start() From 2b405d9177a3fb945b4ee306b6eaf41555ffcd37 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 22 Jun 2017 13:29:07 -0600 Subject: [PATCH 08/18] addressing comments --- qiita_pet/static/css/style.css | 6 +- qiita_pet/templates/list_studies.html | 109 ++++++++++++-------------- 2 files changed, 56 insertions(+), 59 deletions(-) diff --git a/qiita_pet/static/css/style.css b/qiita_pet/static/css/style.css index a39f550d9..82e586ef1 100644 --- a/qiita_pet/static/css/style.css +++ b/qiita_pet/static/css/style.css @@ -21,14 +21,16 @@ td.more-info-processing-jobs{ background: url('../img/details_open.png') no-repeat center center; } tr.shown td.more-info-processing-jobs{ + cursor: pointer; background: url('../img/details_close.png') no-repeat center center; } -td.details-control{ +div.details-control{ cursor: pointer; background: url('../img/details_open.png') no-repeat center center; } -tr.shown td.details-control{ +tr.shown div.details-control{ + cursor: pointer; background: url('../img/details_close.png') no-repeat center center; } diff --git a/qiita_pet/templates/list_studies.html b/qiita_pet/templates/list_studies.html index 094ee7b40..e28047eed 100644 --- a/qiita_pet/templates/list_studies.html +++ b/qiita_pet/templates/list_studies.html @@ -5,11 +5,6 @@ - @@ -18,33 +13,27 @@
IDNameData typeTarget SubfragmentProcessed DateAlgorithmProcessing methodParametersFiles
' + info.artifact_id + '' + info.name + '' + info.data_type + '' + info.target_subfragment.join(', ') + '' + info.timestamp + '' + info.name + ' (' + info.artifact_id + ' - ' + info.timestamp.split('.')[0] + ')' + info.data_type + ' (' + info.target_subfragment.join(', ') + ')' + info.algorithm + '' + params + '' + info.files.join(', ') + '' + info.files.join('
') + '