Skip to content

Readbiom master #2129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 5, 2017
Merged

Readbiom master #2129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b5c7fd5
Fixing latlongs (#2120)
josenavas Apr 28, 2017
4ecff71
public studies are being shown in the user own studies
antgonza Apr 29, 2017
cde1294
fix #2069 - adding tests
antgonza Apr 29, 2017
219f479
flake8
antgonza Apr 29, 2017
50bfc4a
Merge pull request #2122 from antgonza/user-kept-public-study
josenavas Apr 29, 2017
0dd5496
Improving study list speed (#2123)
antgonza May 4, 2017
2282e74
fixing download buttons show (#2127)
antgonza May 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion qiita_db/meta_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,8 @@ def get_lat_longs():
WHERE table_name SIMILAR TO 'sample_[0-9]+'
AND table_schema = 'qiita'
AND column_name IN ('latitude', 'longitude')
AND SPLIT_PART(table_name, '_', 2)::int IN %s;"""
AND SPLIT_PART(table_name, '_', 2)::int IN %s
GROUP BY table_name HAVING COUNT(column_name) = 2;"""
qdb.sql_connection.TRN.add(sql, [tuple(portal_table_ids)])

sql = [('SELECT CAST(latitude AS FLOAT), '
Expand Down
144 changes: 51 additions & 93 deletions qiita_db/test/test_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -799,6 +799,20 @@ def test_get_pubmed_ids_from_dois(self):
self.assertEqual(obs, exp)

def test_generate_study_list(self):
# creating a new study to make sure that empty studies are also
# returned
info = {"timeseries_type_id": 1, "metadata_complete": True,
"mixs_compliant": True, "number_samples_collected": 25,
"number_samples_promised": 28, "study_alias": "TST",
"study_description": "Some description of the study goes here",
"study_abstract": "Some abstract goes here",
"emp_person_id": qdb.study.StudyPerson(1),
"principal_investigator_id": qdb.study.StudyPerson(1),
"lab_person_id": qdb.study.StudyPerson(1)}
new_study = qdb.study.Study.create(
qdb.user.User('shared@foo.bar'), 'test_study_1', efo=[1],
info=info)

exp_info = [{
'metadata_complete': True,
'ebi_submission_status': 'submitted',
Expand All @@ -825,126 +839,70 @@ def test_generate_study_list(self):
'Soils'),
'number_samples_collected': 27,
'study_tags': None
}]
obs_info = qdb.util.generate_study_list([1, 2, 3, 4], False)
}, {
'metadata_complete': True,
'ebi_submission_status': 'not submitted', 'publication_pid': [],
'study_abstract': 'Some abstract goes here',
'pi': ('lab_dude@foo.bar', 'LabDude'), 'status': 'sandbox',
'proc_data_info': [], 'study_tags': None, 'shared': [],
'publication_doi': [], 'study_id': new_study.id,
'ebi_study_accession': None, 'study_title': 'test_study_1',
'number_samples_collected': 0}]
obs_info = qdb.util.generate_study_list([1, 2, 3, 4], True)
self.assertEqual(obs_info, exp_info)

qdb.artifact.Artifact(4).visibility = 'public'
exp_info[0]['status'] = 'public'
exp_info[0]['proc_data_info'] = [
{'data_type': '18S',
'algorithm': 'QIIME (Pick closed-reference OTUs)', 'pid': 4,
'processed_date': '2012-10-02 17:30:00',
{'data_type': '18S', 'name': 'BIOM',
'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)',
'pid': 4, 'processed_date': '2012-10-02 17:30:00',
'params': {'similarity': 0.97, 'reference_name': 'Greengenes',
'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000,
'threads': 1, u'sortmerna_coverage': 0.97,
'reference_version': '13_8'},
'samples': ['1.SKB1.640202', '1.SKB2.640194', '1.SKB3.640195',
'1.SKB4.640189', '1.SKB5.640181', '1.SKB6.640176',
'1.SKB7.640196', '1.SKB8.640193', '1.SKB9.640200',
'1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198',
'1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190',
'1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182',
'1.SKM1.640183', '1.SKM2.640199', '1.SKM3.640197',
'1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
'1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192']},
{'data_type': '18S',
'algorithm': 'QIIME (Pick closed-reference OTUs)', 'pid': 5,
'processed_date': '2012-10-02 17:30:00',
'reference_version': '13_8'}},
{'data_type': '18S', 'name': 'BIOM',
'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)',
'pid': 5, 'processed_date': '2012-10-02 17:30:00',
'params': {'similarity': 0.97, 'reference_name': 'Greengenes',
'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000,
'threads': 1, 'sortmerna_coverage': 0.97,
'reference_version': '13_8'},
'samples': ['1.SKB1.640202', '1.SKB2.640194', '1.SKB3.640195',
'1.SKB4.640189', '1.SKB5.640181', '1.SKB6.640176',
'1.SKB7.640196', '1.SKB8.640193', '1.SKB9.640200',
'1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198',
'1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190',
'1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182',
'1.SKM1.640183', '1.SKM2.640199', '1.SKM3.640197',
'1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
'1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192']},
{'data_type': '16S',
'algorithm': 'QIIME (Pick closed-reference OTUs)', 'pid': 6,
'processed_date': '2012-10-02 17:30:00',
'reference_version': '13_8'}},
{'data_type': '16S', 'name': 'BIOM',
'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)',
'pid': 6, 'processed_date': '2012-10-02 17:30:00',
'params': {'similarity': 0.97, 'reference_name': 'Silva',
'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000,
'threads': 1, 'sortmerna_coverage': 0.97,
'reference_version': 'test'},
'samples': ['1.SKB1.640202', '1.SKB2.640194', '1.SKB3.640195',
'1.SKB4.640189', '1.SKB5.640181', '1.SKB6.640176',
'1.SKB7.640196', '1.SKB8.640193', '1.SKB9.640200',
'1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198',
'1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190',
'1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182',
'1.SKM1.640183', '1.SKM2.640199', '1.SKM3.640197',
'1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
'1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192']}]
obs_info = qdb.util.generate_study_list([1, 2, 3, 4], True,
public_only=True)
'reference_version': 'test'}}]
obs_info = qdb.util.generate_study_list([1, 2, 3, 4], True)
self.assertEqual(obs_info, exp_info)

exp_info[0]['proc_data_info'] = [
{'data_type': '18S',
'algorithm': 'QIIME (Pick closed-reference OTUs)', 'pid': 4,
'processed_date': '2012-10-02 17:30:00',
{'data_type': '18S', 'name': 'BIOM',
'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)',
'pid': 4, 'processed_date': '2012-10-02 17:30:00',
'params': {'similarity': 0.97, 'reference_name': 'Greengenes',
'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000,
'threads': 1, 'sortmerna_coverage': 0.97,
'reference_version': '13_8'},
'samples': ['1.SKB1.640202', '1.SKB2.640194', '1.SKB3.640195',
'1.SKB4.640189', '1.SKB5.640181', '1.SKB6.640176',
'1.SKB7.640196', '1.SKB8.640193', '1.SKB9.640200',
'1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198',
'1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190',
'1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182',
'1.SKM1.640183', '1.SKM2.640199', '1.SKM3.640197',
'1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
'1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192']},
{'data_type': '18S',
'algorithm': 'QIIME (Pick closed-reference OTUs)', 'pid': 5,
'processed_date': '2012-10-02 17:30:00',
'reference_version': '13_8'}},
{'data_type': '18S', 'name': 'BIOM',
'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)',
'pid': 5, 'processed_date': '2012-10-02 17:30:00',
'params': {'similarity': 0.97, 'reference_name': 'Greengenes',
'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000,
'threads': 1, 'sortmerna_coverage': 0.97,
'reference_version': '13_8'},
'samples': ['1.SKB1.640202', '1.SKB2.640194', '1.SKB3.640195',
'1.SKB4.640189', '1.SKB5.640181', '1.SKB6.640176',
'1.SKB7.640196', '1.SKB8.640193', '1.SKB9.640200',
'1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198',
'1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190',
'1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182',
'1.SKM1.640183', '1.SKM2.640199', '1.SKM3.640197',
'1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
'1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192']},
{'data_type': '16S',
'algorithm': 'QIIME (Pick closed-reference OTUs)', 'pid': 6,
'processed_date': '2012-10-02 17:30:00',
'reference_version': '13_8'}},
{'data_type': '16S', 'name': 'BIOM',
'algorithm': 'QIIME v1.9.1 (Pick closed-reference OTUs)',
'pid': 6, 'processed_date': '2012-10-02 17:30:00',
'params': {'similarity': 0.97, 'reference_name': 'Silva',
'sortmerna_e_value': 1, u'sortmerna_max_pos': 10000,
'threads': 1, 'sortmerna_coverage': 0.97,
'reference_version': 'test'},
'samples': ['1.SKB1.640202', '1.SKB2.640194', '1.SKB3.640195',
'1.SKB4.640189', '1.SKB5.640181', '1.SKB6.640176',
'1.SKB7.640196', '1.SKB8.640193', '1.SKB9.640200',
'1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198',
'1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190',
'1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182',
'1.SKM1.640183', '1.SKM2.640199', '1.SKM3.640197',
'1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
'1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192']},
{'processed_date': '2012-10-02 17:30:00', 'pid': 7,
'samples': ['1.SKB1.640202', '1.SKB2.640194', '1.SKB3.640195',
'1.SKB4.640189', '1.SKB5.640181', '1.SKB6.640176',
'1.SKB7.640196', '1.SKB8.640193', '1.SKB9.640200',
'1.SKD1.640179', '1.SKD2.640178', '1.SKD3.640198',
'1.SKD4.640185', '1.SKD5.640186', '1.SKD6.640190',
'1.SKD7.640191', '1.SKD8.640184', '1.SKD9.640182',
'1.SKM1.640183', '1.SKM2.640199', '1.SKM3.640197',
'1.SKM4.640180', '1.SKM5.640177', '1.SKM6.640187',
'1.SKM7.640188', '1.SKM8.640201', '1.SKM9.640192'],
'reference_version': 'test'}},
{'processed_date': '2012-10-02 17:30:00', 'pid': 7, 'name': 'BIOM',
'data_type': '16S'}]
obs_info = qdb.util.generate_study_list([1, 2, 3, 4], True)
obs_info = qdb.util.generate_study_list([1, 2, 3, 4], False)
self.assertEqual(obs_info, exp_info)


Expand Down
67 changes: 38 additions & 29 deletions qiita_db/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1266,16 +1266,13 @@ def supported_filepath_types(artifact_type):
return qdb.sql_connection.TRN.execute_fetchindex()


def generate_study_list(study_ids, build_samples, public_only=False):
def generate_study_list(study_ids, public_only=False):
"""Get general study information

Parameters
----------
study_ids : list of ints
The study ids to look for. Non-existing ids will be ignored
build_samples : bool
If true the sample information for each process artifact within each
study will be included
public_only : bool, optional
If true, return only public BIOM artifacts. Default: false.

Expand Down Expand Up @@ -1334,6 +1331,13 @@ def generate_study_list(study_ids, build_samples, public_only=False):
LEFT JOIN qiita.artifact_type USING (artifact_type_id)
WHERE artifact_type='BIOM' AND
study_id = qiita.study.study_id) AS artifact_biom_ts,
- all the BIOM names sorted by artifact_id that belong to the study
(SELECT array_agg(name ORDER BY artifact_id)
FROM qiita.study_artifact
LEFT JOIN qiita.artifact USING (artifact_id)
LEFT JOIN qiita.artifact_type USING (artifact_type_id)
WHERE artifact_type='BIOM' AND
study_id = qiita.study.study_id) AS artifact_biom_name,
- all the BIOM visibility sorted by artifact_id that belong to the study
(SELECT array_agg(visibility ORDER BY artifact_id)
FROM qiita.study_artifact
Expand Down Expand Up @@ -1408,6 +1412,12 @@ def generate_study_list(study_ids, build_samples, public_only=False):
LEFT JOIN qiita.artifact_type USING (artifact_type_id)
WHERE artifact_type='BIOM' AND
study_id = qiita.study.study_id) AS artifact_biom_ts,
(SELECT array_agg(name ORDER BY artifact_id)
FROM qiita.study_artifact
LEFT JOIN qiita.artifact USING (artifact_id)
LEFT JOIN qiita.artifact_type USING (artifact_type_id)
WHERE artifact_type='BIOM' AND
study_id = qiita.study.study_id) AS artifact_biom_name,
(SELECT array_agg(visibility ORDER BY artifact_id)
FROM qiita.study_artifact
LEFT JOIN qiita.artifact USING (artifact_id)
Expand Down Expand Up @@ -1481,17 +1491,19 @@ def generate_study_list(study_ids, build_samples, public_only=False):
del info["shared_with_email"]

info['proc_data_info'] = []
if build_samples and info['artifact_biom_ids']:
if info['artifact_biom_ids']:
to_loop = zip(
info['artifact_biom_ids'], info['artifact_biom_dts'],
info['artifact_biom_ts'], info['artifact_biom_params'],
info['artifact_biom_cmd'], info['artifact_biom_vis'])
for artifact_id, dt, ts, params, cmd, vis in to_loop:
info['artifact_biom_cmd'], info['artifact_biom_vis'],
info['artifact_biom_name'])
for artifact_id, dt, ts, params, cmd, vis, name in to_loop:
if public_only and vis != 'public':
continue
proc_info = {'processed_date': str(ts)}
proc_info['pid'] = artifact_id
proc_info['data_type'] = dt
proc_info['name'] = name

# if cmd exists then we can get its parameters
if cmd is not None:
Expand All @@ -1503,6 +1515,7 @@ def generate_study_list(study_ids, build_samples, public_only=False):
'del_keys': [k for k, v in viewitems(
c.parameters) if v[0] == 'artifact'],
'sfwn': c.software.name,
'sfv': c.software.version,
'cmdn': c.name
}
for k in commands[cmd]['del_keys']:
Expand All @@ -1526,31 +1539,27 @@ def generate_study_list(study_ids, build_samples, public_only=False):
params['reference_version'] = refs[rid][
'version']

proc_info['algorithm'] = '%s (%s)' % (
commands[cmd]['sfwn'], commands[cmd]['cmdn'])
proc_info['algorithm'] = '%s v%s (%s)' % (
commands[cmd]['sfwn'], commands[cmd]['sfv'],
commands[cmd]['cmdn'])
proc_info['params'] = params

# getting all samples
sql = """SELECT sample_id from qiita.prep_template_sample
WHERE prep_template_id = (
SELECT prep_template_id
FROM qiita.prep_template
WHERE artifact_id IN (
SELECT *
FROM qiita.find_artifact_roots(%s)))"""
qdb.sql_connection.TRN.add(sql, [proc_info['pid']])
proc_info['samples'] = sorted(
qdb.sql_connection.TRN.execute_fetchflatten())

info["proc_data_info"].append(proc_info)

del info["artifact_biom_ids"]
del info["artifact_biom_dts"]
del info["artifact_biom_ts"]
del info["artifact_biom_params"]
del info['artifact_biom_cmd']
del info['artifact_biom_vis']

infolist.append(info)
infolist.append({
'metadata_complete': info['metadata_complete'],
'publication_pid': info['publication_pid'],
'ebi_submission_status': info['ebi_submission_status'],
'shared': info['shared'],
'study_abstract': info['study_abstract'], 'pi': info['pi'],
'status': info['status'],
'proc_data_info': info['proc_data_info'],
'study_tags': info['study_tags'],
'publication_doi': info['publication_doi'],
'study_id': info['study_id'],
'ebi_study_accession': info['ebi_study_accession'],
'study_title': info['study_title'],
'number_samples_collected': info['number_samples_collected']
})

return infolist
4 changes: 2 additions & 2 deletions qiita_pet/handlers/api_proxy/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
from .artifact import (artifact_graph_get_req, artifact_types_get_req,
artifact_post_req, artifact_get_req,
artifact_status_put_req, artifact_delete_req,
artifact_summary_get_request,
artifact_summary_get_request, artifact_get_prep_req,
artifact_summary_post_request, artifact_patch_request)
from .ontology import ontology_patch_handler
from .processing import (
Expand All @@ -48,7 +48,7 @@
'study_get_req', 'sample_template_summary_get_req',
'sample_template_delete_req', 'sample_template_filepaths_get_req',
'prep_template_summary_get_req', 'prep_template_post_req',
'prep_template_delete_req',
'prep_template_delete_req', 'artifact_get_prep_req',
'prep_template_graph_get_req', 'prep_template_filepaths_get_req',
'artifact_get_req', 'artifact_status_put_req',
'artifact_delete_req', 'prep_template_get_req', 'study_delete_req',
Expand Down
Loading