From 7d4ad951f805f629dff915153a9da861b74da23a Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Mon, 30 Aug 2021 08:22:44 -0700 Subject: [PATCH 1/2] RESTful sample status (#3139) * TST: tests for the sample status end points * API: add sample status endpoints * Defensive assertion on detail maker * Address @antgonza's comments * Limit memory use when caching prep info --- qiita_pet/handlers/rest/__init__.py | 7 +- qiita_pet/handlers/rest/study_samples.py | 110 +++++++++++++++++ qiita_pet/test/rest/test_sample_detail.py | 141 ++++++++++++++++++++++ 3 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 qiita_pet/test/rest/test_sample_detail.py diff --git a/qiita_pet/handlers/rest/__init__.py b/qiita_pet/handlers/rest/__init__.py index cfd842dc7..73ad9382a 100644 --- a/qiita_pet/handlers/rest/__init__.py +++ b/qiita_pet/handlers/rest/__init__.py @@ -8,7 +8,9 @@ from .study import StudyHandler, StudyCreatorHandler, StudyStatusHandler from .study_samples import (StudySamplesHandler, StudySamplesInfoHandler, - StudySamplesCategoriesHandler) + StudySamplesCategoriesHandler, + StudySamplesDetailHandler, + StudySampleDetailHandler) from .study_person import StudyPersonHandler from .study_preparation import (StudyPrepCreatorHandler, StudyPrepArtifactCreatorHandler) @@ -26,6 +28,9 @@ (r"/api/v1/study/([0-9]+)/samples/categories=([a-zA-Z\-0-9\.:,_]*)", StudySamplesCategoriesHandler), (r"/api/v1/study/([0-9]+)/samples", StudySamplesHandler), + (r"/api/v1/study/([0-9]+)/samples/status", StudySamplesDetailHandler), + (r"/api/v1/study/([0-9]+)/sample/([a-zA-Z\-0-9\.]+)/status", + StudySampleDetailHandler), (r"/api/v1/study/([0-9]+)/samples/info", StudySamplesInfoHandler), (r"/api/v1/person(.*)", StudyPersonHandler), (r"/api/v1/study/([0-9]+)/preparation/([0-9]+)/artifact", diff --git a/qiita_pet/handlers/rest/study_samples.py b/qiita_pet/handlers/rest/study_samples.py index de2424926..0679b3b57 100644 --- a/qiita_pet/handlers/rest/study_samples.py +++ b/qiita_pet/handlers/rest/study_samples.py @@ -5,6 +5,8 @@ # # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- +from collections import defaultdict + from tornado.escape import json_encode, json_decode import pandas as pd @@ -12,6 +14,114 @@ from .rest_handler import RESTHandler +def _sample_details(study, samples): + def detail_maker(**kwargs): + base = {'sample_id': None, + 'sample_found': False, + 'ebi_sample_accession': None, + 'preparation_id': None, + 'ebi_experiment_accession': None, + 'preparation_visibility': None, + 'preparation_type': None} + + assert set(kwargs).issubset(set(base)), "Unexpected key to set" + + base.update(kwargs) + return base + + # cache sample detail for lookup + study_samples = set(study.sample_template) + sample_accessions = study.sample_template.ebi_sample_accessions + + # cache preparation information that we'll need + + # map of {sample_id: [indices, of, light, prep, info, ...]} + sample_prep_mapping = defaultdict(list) + pt_light = [] + offset = 0 + incoming_samples = set(samples) + for pt in study.prep_templates(): + prep_samples = set(pt) + overlap = incoming_samples & prep_samples + + if overlap: + # cache if any of or query samples are present on the prep + + # reduce accessions to only samples of interest + accessions = pt.ebi_experiment_accessions + overlap_accessions = {i: accessions[i] for i in overlap} + + # store the detail we need + pt_light.append((pt.id, overlap_accessions, + pt.status, pt.data_type())) + + # only care about mapping the incoming samples + for ptsample in overlap: + sample_prep_mapping[ptsample].append(offset) + + offset += 1 + + details = [] + for sample in samples: + if sample in study_samples: + # if the sample exists + sample_acc = sample_accessions.get(sample) + + if sample in sample_prep_mapping: + # if the sample is present in any prep, pull out the detail + # specific to those preparations + for pt_idx in sample_prep_mapping[sample]: + ptid, ptacc, ptstatus, ptdtype = pt_light[pt_idx] + + details.append(detail_maker( + sample_id=sample, + sample_found=True, + ebi_sample_accession=sample_acc, + preparation_id=ptid, + ebi_experiment_accession=ptacc.get(sample), + preparation_visibility=ptstatus, + preparation_type=ptdtype)) + else: + # the sample is not present on any preparations + details.append(detail_maker( + sample_id=sample, + sample_found=True, + + # it would be weird to have an EBI sample accession + # but not be present on a preparation...? + ebi_sample_accession=sample_acc)) + else: + # the is not present, let's note and move ona + details.append(detail_maker(sample_id=sample)) + + return details + + +class StudySampleDetailHandler(RESTHandler): + @authenticate_oauth + def get(self, study_id, sample_id): + study = self.safe_get_study(study_id) + sample_detail = _sample_details(study, [sample_id, ]) + self.write(json_encode(sample_detail)) + self.finish() + + +class StudySamplesDetailHandler(RESTHandler): + @authenticate_oauth + def post(self, study_id): + samples = json_decode(self.request.body) + + if 'sample_ids' not in samples: + self.fail('Missing sample_id key', 400) + return + + study = self.safe_get_study(study_id) + samples_detail = _sample_details(study, samples['sample_ids']) + + self.write(json_encode(samples_detail)) + self.finish() + + class StudySamplesHandler(RESTHandler): @authenticate_oauth diff --git a/qiita_pet/test/rest/test_sample_detail.py b/qiita_pet/test/rest/test_sample_detail.py new file mode 100644 index 000000000..33e96276d --- /dev/null +++ b/qiita_pet/test/rest/test_sample_detail.py @@ -0,0 +1,141 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from unittest import main, TestCase + +from tornado.escape import json_decode + +import qiita_db + +from qiita_pet.test.rest.test_base import RESTHandlerTestCase +from qiita_pet.handlers.rest.study_samples import _sample_details + + +class SupportTests(TestCase): + def test_samples_detail(self): + exp = [{'sample_id': '1.SKD7.640191', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000021', + 'preparation_id': 1, + 'ebi_experiment_accession': 'ERX0000021', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}, + {'sample_id': '1.SKD7.640191', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000021', + 'preparation_id': 2, + 'ebi_experiment_accession': 'ERX0000021', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}, + {'sample_id': 'doesnotexist', + 'sample_found': False, + 'ebi_sample_accession': None, + 'preparation_id': None, + 'ebi_experiment_accession': None, + 'preparation_visibility': None, + 'preparation_type': None}] + obs = _sample_details(qiita_db.study.Study(1), + ['1.SKD7.640191', 'doesnotexist']) + self.assertEqual(len(obs), len(exp)) + self.assertEqual(obs, exp) + + +class SampleDetailHandlerTests(RESTHandlerTestCase): + def test_get_missing_sample(self): + exp = [{'sample_id': 'doesnotexist', + 'sample_found': False, + 'ebi_sample_accession': None, + 'preparation_id': None, + 'ebi_experiment_accession': None, + 'preparation_visibility': None, + 'preparation_type': None}, ] + + response = self.get('/api/v1/study/1/sample/doesnotexist/status', + headers=self.headers) + self.assertEqual(response.code, 200) + obs = json_decode(response.body) + self.assertEqual(obs, exp) + + def test_get_valid_sample(self): + exp = [{'sample_id': '1.SKD7.640191', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000021', + 'preparation_id': 1, + 'ebi_experiment_accession': 'ERX0000021', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}, + {'sample_id': '1.SKD7.640191', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000021', + 'preparation_id': 2, + 'ebi_experiment_accession': 'ERX0000021', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}] + + response = self.get('/api/v1/study/1/sample/1.SKD7.640191/status', + headers=self.headers) + self.assertEqual(response.code, 200) + obs = json_decode(response.body) + self.assertEqual(obs, exp) + + def test_post_samples_status_bad_request(self): + body = {'malformed': 'with garbage'} + response = self.post('/api/v1/study/1/samples/status', + headers=self.headers, + data=body, asjson=True) + self.assertEqual(response.code, 400) + + def test_post_samples_status(self): + exp = [{'sample_id': '1.SKD7.640191', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000021', + 'preparation_id': 1, + 'ebi_experiment_accession': 'ERX0000021', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}, + {'sample_id': '1.SKD7.640191', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000021', + 'preparation_id': 2, + 'ebi_experiment_accession': 'ERX0000021', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}, + {'sample_id': 'doesnotexist', + 'sample_found': False, + 'ebi_sample_accession': None, + 'preparation_id': None, + 'ebi_experiment_accession': None, + 'preparation_visibility': None, + 'preparation_type': None}, + {'sample_id': '1.SKM5.640177', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000005', + 'preparation_id': 1, + 'ebi_experiment_accession': 'ERX0000005', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}, + {'sample_id': '1.SKM5.640177', + 'sample_found': True, + 'ebi_sample_accession': 'ERS000005', + 'preparation_id': 2, + 'ebi_experiment_accession': 'ERX0000005', + 'preparation_visibility': 'private', + 'preparation_type': '18S'}] + + body = {'sample_ids': ['1.SKD7.640191', 'doesnotexist', + '1.SKM5.640177']} + response = self.post('/api/v1/study/1/samples/status', + headers=self.headers, + data=body, asjson=True) + self.assertEqual(response.code, 200) + obs = json_decode(response.body) + self.assertEqual(obs, exp) + + +if __name__ == '__main__': + main() From dcf6cbe620de9e369b075838494f0d557124696d Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 8 Sep 2021 11:42:32 -0600 Subject: [PATCH 2/2] install is broken (#3143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix build * update pip * rm pip install * conda version * conda autoupdate default * adding @ElDeveloper comments * setuptools==58.0.3 * + sphinx==4.1.2 * Update .github/workflows/qiita-ci.yml * Update .github/workflows/qiita-ci.yml Co-authored-by: Yoshiki Vázquez Baeza --- .github/workflows/qiita-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index ab99c7d9c..9ef3275db 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -65,6 +65,8 @@ jobs: conda config --add channels conda-forge conda create -q --yes -n qiita python=3.6 pip libgfortran numpy nginx cython redis conda activate qiita + pip install -U pip + pip install 'setuptools<=58.0.1' pip install sphinx sphinx-bootstrap-theme nose-timer Click coverage # Configuring SSH