Skip to content

Commit e094701

Browse files
josenavasantgonza
josenavas
authored and
antgonza
committed
Fixes Qiita db (#2046)
* New DB structure * Adding python patch * Adding a biom so we can actually execute the patch * Fixing the patch to correctly transfer the information from the old structure to the new one * Fixing patch * Fixing patch and a few other bits to make the patch run successfully * These files are no longer needed * Removing unused code * Droping analysis status table * Linking the analysis with all the artifacts * Fixing typo * Fixing HTML and dbschema files * Adding analyisis jobs * Extending the artifact to work with the analysis * Allowing multiomics datatype * Adding private_job_submitter and modifying proc job handler to use it * Adding logging column to the analysis * Adding datatype to the analysis-processing job table * Adding REST endpoint to access the analysis metadata * Adding private jobs to plugin * Fixing typo * Fixing the processing jobs complete * Removing the old job code * Oops removed the wrong file * Removing QiitaStatusObject because it is not used * fixing metautil * Fixing porntal, setup and sql tests * Fixing user and util * Fixing qiita_db * Addressing @antgonza's comments * Taking into account non-phylogenetic metrics in beta diversity * Addressing @antgonza's comments
1 parent 23abbdb commit e094701

File tree

14 files changed

+32
-282
lines changed

14 files changed

+32
-282
lines changed

‎qiita_db/base.py

Lines changed: 0 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
:toctree: generated/
1515
1616
QiitaObject
17-
QiitaStatusObject
1817
"""
1918

2019
# -----------------------------------------------------------------------------
@@ -220,106 +219,3 @@ def __hash__(self):
220219
def id(self):
221220
r"""The object id on the storage system"""
222221
return self._id
223-
224-
225-
class QiitaStatusObject(QiitaObject):
226-
r"""Base class for any qiita_db object with a status property
227-
228-
Attributes
229-
----------
230-
status
231-
232-
Methods
233-
-------
234-
check_status
235-
_status_setter_checks
236-
"""
237-
238-
@property
239-
def status(self):
240-
r"""String with the current status of the analysis"""
241-
# Get the DB status of the object
242-
with qdb.sql_connection.TRN:
243-
sql = """SELECT status FROM qiita.{0}_status
244-
WHERE {0}_status_id = (
245-
SELECT {0}_status_id FROM qiita.{0}
246-
WHERE {0}_id = %s)""".format(self._table)
247-
qdb.sql_connection.TRN.add(sql, [self._id])
248-
return qdb.sql_connection.TRN.execute_fetchlast()
249-
250-
def _status_setter_checks(self):
251-
r"""Perform any extra checks that needed to be done before setting the
252-
object status on the database. Should be overwritten by the subclasses
253-
"""
254-
raise qdb.exceptions.QiitaDBNotImplementedError()
255-
256-
@status.setter
257-
def status(self, status):
258-
r"""Change the status of the analysis
259-
260-
Parameters
261-
----------
262-
status: str
263-
The new object status
264-
"""
265-
with qdb.sql_connection.TRN:
266-
# Perform any extra checks needed before
267-
# we update the status in the DB
268-
self._status_setter_checks()
269-
270-
# Update the status of the object
271-
sql = """UPDATE qiita.{0} SET {0}_status_id = (
272-
SELECT {0}_status_id FROM qiita.{0}_status
273-
WHERE status = %s)
274-
WHERE {0}_id = %s""".format(self._table)
275-
qdb.sql_connection.TRN.add(sql, [status, self._id])
276-
qdb.sql_connection.TRN.execute()
277-
278-
def check_status(self, status, exclude=False):
279-
r"""Checks status of object.
280-
281-
Parameters
282-
----------
283-
status: iterable
284-
Iterable of statuses to check against.
285-
exclude: bool, optional
286-
If True, will check that database status is NOT one of the statuses
287-
passed. Default False.
288-
289-
Returns
290-
-------
291-
bool
292-
True if the object status is in the desired set of statuses. False
293-
otherwise.
294-
295-
Notes
296-
-----
297-
This assumes the following database setup is in place: For a given
298-
cls._table setting, such as "table", there is a corresponding table
299-
with the name "table_status" holding the status entries allowed. This
300-
table has a column called "status" that holds the values corresponding
301-
to what is passed as status in this function and a column
302-
"table_status_id" corresponding to the column of the same name in
303-
"table".
304-
305-
Table setup:
306-
foo: foo_status_id ----> foo_status: foo_status_id, status
307-
"""
308-
with qdb.sql_connection.TRN:
309-
# Get all available statuses
310-
sql = "SELECT DISTINCT status FROM qiita.{0}_status".format(
311-
self._table)
312-
qdb.sql_connection.TRN.add(sql)
313-
# We need to access to the results of the last SQL query,
314-
# hence indexing using -1
315-
avail_status = [
316-
x[0] for x in qdb.sql_connection.TRN.execute_fetchindex()]
317-
318-
# Check that all the provided status are valid status
319-
if set(status).difference(avail_status):
320-
raise ValueError("%s are not valid status values"
321-
% set(status).difference(avail_status))
322-
323-
# Get the DB status of the object
324-
dbstatus = self.status
325-
return dbstatus not in status if exclude else dbstatus in status

‎qiita_db/investigation.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
REQUIRED_KEYS = {"name", "description", "contact_person"}
2424

2525

26-
class Investigation(qdb.base.QiitaStatusObject):
26+
class Investigation(qdb.base.QiitaObject):
2727
"""
2828
Study object to access to the Qiita Study information
2929

‎qiita_db/meta_util.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,17 +109,24 @@ def get_accessible_filepath_ids(user):
109109
filepath_ids.update({fid for fid, _ in pt.get_filepaths()})
110110

111111
# Then add the filepaths of the sample template
112-
filepath_ids.update(
113-
{fid
114-
for fid, _ in artifact.study.sample_template.get_filepaths()})
112+
study = artifact.study
113+
if study:
114+
filepath_ids.update(
115+
{fid
116+
for fid, _ in study.sample_template.get_filepaths()})
115117

116118
# Next, analyses
117119
# Same as before, there are public, private, and shared
118120
analyses = qdb.analysis.Analysis.get_by_status('public') | \
119121
user.private_analyses | user.shared_analyses
120122

121-
for analysis in analyses:
122-
filepath_ids.update(analysis.all_associated_filepath_ids)
123+
if analyses:
124+
sql = """SELECT filepath_id
125+
FROM qiita.analysis_filepath
126+
WHERE analysis_id IN %s"""
127+
sql_args = tuple([a.id for a in analyses])
128+
qdb.sql_connection.TRN.add(sql, [sql_args])
129+
filepath_ids.update(qdb.sql_connection.TRN.execute_fetchflatten())
123130

124131
return filepath_ids
125132

‎qiita_db/metadata_template/test/test_prep_template.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -876,8 +876,6 @@ def _common_creation_checks(self, pt, fp_count):
876876
# prep and qiime files have been created
877877
filepaths = pt.get_filepaths()
878878
self.assertEqual(len(filepaths), 2)
879-
self.assertEqual(filepaths[0][0], fp_count + 2)
880-
self.assertEqual(filepaths[1][0], fp_count + 1)
881879

882880
def test_create(self):
883881
"""Creates a new PrepTemplate"""
@@ -998,8 +996,6 @@ def test_create_warning(self):
998996
# prep and qiime files have been created
999997
filepaths = pt.get_filepaths()
1000998
self.assertEqual(len(filepaths), 2)
1001-
self.assertEqual(filepaths[0][0], fp_count + 2)
1002-
self.assertEqual(filepaths[1][0], fp_count + 1)
1003999

10041000
def test_create_investigation_type_error(self):
10051001
"""Create raises an error if the investigation_type does not exists"""

‎qiita_db/metadata_template/test/test_sample_template.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1320,7 +1320,7 @@ def test_get_filepath(self):
13201320
# change based on time and the same functionality is being tested
13211321
# in data.py
13221322
exp_id = self.conn_handler.execute_fetchone(
1323-
"SELECT count(1) FROM qiita.filepath")[0] + 1
1323+
"SELECT last_value FROM qiita.filepath_filepath_id_seq")[0] + 1
13241324
st = qdb.metadata_template.sample_template.SampleTemplate.create(
13251325
self.metadata, self.new_study)
13261326
self.assertEqual(st.get_filepaths()[0][0], exp_id)

‎qiita_db/portal.py

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -91,14 +91,10 @@ def create(cls, portal, desc):
9191
SELECT email FROM qiita.qiita_user
9292
LOOP
9393
INSERT INTO qiita.analysis
94-
(email, name, description, dflt,
95-
analysis_status_id)
96-
VALUES (eml, eml || '-dflt', 'dflt', true, 1)
94+
(email, name, description, dflt)
95+
VALUES (eml, eml || '-dflt', 'dflt', true)
9796
RETURNING analysis_id INTO aid;
9897
99-
INSERT INTO qiita.analysis_workflow (analysis_id, step)
100-
VALUES (aid, 2);
101-
10298
INSERT INTO qiita.analysis_portal
10399
(analysis_id, portal_type_id)
104100
VALUES (aid, pid);
@@ -162,9 +158,6 @@ def delete(portal):
162158
DELETE FROM qiita.analysis_portal
163159
WHERE analysis_id = aid;
164160
165-
DELETE FROM qiita.analysis_workflow
166-
WHERE analysis_id = aid;
167-
168161
DELETE FROM qiita.analysis_sample
169162
WHERE analysis_id = aid;
170163

‎qiita_db/test/test_base.py

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -84,59 +84,5 @@ def test_not_equal_type(self):
8484
self.assertNotEqual(self.tester, new)
8585

8686

87-
@qiita_test_checker()
88-
class QiitaStatusObjectTest(TestCase):
89-
"""Tests that the QiitaStatusObject class functions act correctly"""
90-
91-
def setUp(self):
92-
# We need an actual subclass in order to test the equality functions
93-
self.tester = qdb.analysis.Analysis(1)
94-
95-
def test_status(self):
96-
"""Correctly returns the status of the object"""
97-
self.assertEqual(self.tester.status, "in_construction")
98-
99-
def test_check_status_single(self):
100-
"""check_status works passing a single status"""
101-
self.assertTrue(self.tester.check_status(["in_construction"]))
102-
self.assertFalse(self.tester.check_status(["queued"]))
103-
104-
def test_check_status_exclude_single(self):
105-
"""check_status works passing a single status and the exclude flag"""
106-
self.assertTrue(self.tester.check_status(["public"], exclude=True))
107-
self.assertFalse(self.tester.check_status(["in_construction"],
108-
exclude=True))
109-
110-
def test_check_status_list(self):
111-
"""check_status work passing a list of status"""
112-
self.assertTrue(self.tester.check_status(
113-
["in_construction", "queued"]))
114-
self.assertFalse(self.tester.check_status(
115-
["public", "queued"]))
116-
117-
def test_check_status_exclude_list(self):
118-
"""check_status work passing a list of status and the exclude flag"""
119-
self.assertTrue(self.tester.check_status(
120-
["public", "queued"], exclude=True))
121-
self.assertFalse(self.tester.check_status(
122-
["in_construction", "queued"], exclude=True))
123-
124-
def test_check_status_unknown_status(self):
125-
"""check_status raises an error if an invalid status is provided"""
126-
with self.assertRaises(ValueError):
127-
self.tester.check_status(["foo"])
128-
129-
with self.assertRaises(ValueError):
130-
self.tester.check_status(["foo"], exclude=True)
131-
132-
def test_check_status_unknown_status_list(self):
133-
"""check_status raises an error if an invalid status list is provided
134-
"""
135-
with self.assertRaises(ValueError):
136-
self.tester.check_status(["foo", "bar"])
137-
138-
with self.assertRaises(ValueError):
139-
self.tester.check_status(["foo", "bar"], exclude=True)
140-
14187
if __name__ == '__main__':
14288
main()

‎qiita_db/test/test_meta_util.py

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,13 @@ def test_get_accessible_filepath_ids(self):
4646
obs = qdb.meta_util.get_accessible_filepath_ids(
4747
qdb.user.User('shared@foo.bar'))
4848
self.assertItemsEqual(obs, {
49-
1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21})
49+
1, 2, 3, 4, 5, 9, 12, 16, 17, 18, 19, 20, 21})
5050

5151
# Now shared should not have access to the study files
5252
self._unshare_studies()
5353
obs = qdb.meta_util.get_accessible_filepath_ids(
5454
qdb.user.User('shared@foo.bar'))
55-
self.assertItemsEqual(obs, {16, 14, 15, 13})
55+
self.assertItemsEqual(obs, {16})
5656

5757
# Now shared should not have access to any files
5858
self._unshare_analyses()
@@ -64,10 +64,11 @@ def test_get_accessible_filepath_ids(self):
6464
self._set_artifact_public()
6565
obs = qdb.meta_util.get_accessible_filepath_ids(
6666
qdb.user.User('shared@foo.bar'))
67-
self.assertEqual(obs, {1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21})
67+
self.assertEqual(
68+
obs, {1, 2, 3, 4, 5, 9, 12, 15, 16, 17, 18, 19, 20, 21, 22})
6869

6970
# Test that it doesn't break: if the SampleTemplate hasn't been added
70-
exp = {1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
71+
exp = {1, 2, 3, 4, 5, 9, 12, 15, 16, 17, 18, 19, 20, 21, 22}
7172
obs = qdb.meta_util.get_accessible_filepath_ids(
7273
qdb.user.User('test@foo.bar'))
7374
self.assertEqual(obs, exp)
@@ -99,9 +100,11 @@ def test_get_accessible_filepath_ids(self):
99100
self.assertEqual(obs, exp)
100101

101102
# admin should have access to everything
102-
count = self.conn_handler.execute_fetchone("SELECT count(*) FROM "
103-
"qiita.filepath")[0]
103+
count = self.conn_handler.execute_fetchone(
104+
"SELECT last_value FROM qiita.filepath_filepath_id_seq")[0]
104105
exp = set(range(1, count + 1))
106+
exp.discard(13)
107+
exp.discard(14)
105108
obs = qdb.meta_util.get_accessible_filepath_ids(
106109
qdb.user.User('admin@foo.bar'))
107110
self.assertEqual(obs, exp)

‎qiita_db/test/test_portal.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,6 @@ def test_remove_portal(self):
7272
qdb.portal.Portal.delete("QIITA")
7373

7474
qdb.portal.Portal.create("NEWPORTAL2", "SOMEDESC")
75-
# Add analysis to this new portal and make sure error raised
76-
qiita_config.portal = "NEWPORTAL2"
77-
qdb.analysis.Analysis.create(
78-
qdb.user.User("test@foo.bar"), "newportal analysis", "desc")
79-
qiita_config.portal = "QIITA"
80-
with self.assertRaises(qdb.exceptions.QiitaDBError):
81-
qdb.portal.Portal.delete("NEWPORTAL2")
8275

8376
# Add study to this new portal and make sure error raised
8477
info = {

‎qiita_db/test/test_setup.py

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -62,18 +62,9 @@ def test_prep_1(self):
6262
def test_reference(self):
6363
self.assertEqual(get_count("qiita.reference"), 2)
6464

65-
def test_job(self):
66-
self.assertEqual(get_count("qiita.job"), 3)
67-
6865
def test_analysis(self):
6966
self.assertEqual(get_count("qiita.analysis"), 10)
7067

71-
def test_analysis_job(self):
72-
self.assertEqual(get_count("qiita.analysis_job"), 3)
73-
74-
def test_analysis_workflow(self):
75-
self.assertEqual(get_count("qiita.analysis_workflow"), 2)
76-
7768
def test_analysis_filepath(self):
7869
self.assertEqual(get_count("qiita.analysis_filepath"), 2)
7970

@@ -83,12 +74,6 @@ def test_analysis_sample(self):
8374
def test_analysis_users(self):
8475
self.assertEqual(get_count("qiita.analysis_users"), 1)
8576

86-
def test_job_results_filepath(self):
87-
self.assertEqual(get_count("qiita.job_results_filepath"), 2)
88-
89-
def test_command_data_type(self):
90-
self.assertEqual(get_count("qiita.command_data_type"), 14)
91-
9277
def test_ontology(self):
9378
self.assertTrue(check_count('qiita.ontology', 1))
9479

0 commit comments

Comments
 (0)