Skip to content

Sample generate files improvements #2749

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Dec 12, 2018
Merged

Sample generate files improvements #2749

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8d8f674
sample info file generate_files is now "smart"
antgonza Dec 6, 2018
9401943
upstream dev
antgonza Dec 11, 2018
26dca2e
fix test
antgonza Dec 11, 2018
b0dc603
Merge branch 'dev' of https://github.com/biocore/qiita into sample-ge…
antgonza Dec 11, 2018
a93ca98
addressing @charles-cowart comments
antgonza Dec 12, 2018
9e5c964
fix error
antgonza Dec 12, 2018
1005f85
fix error
antgonza Dec 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 40 additions & 11 deletions qiita_db/metadata_template/base_metadata_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -696,7 +696,7 @@ def _common_delete_sample_steps(self, sample_names):
qdb.sql_connection.TRN.add(sql2, [sn, self.id])
qdb.sql_connection.TRN.execute()

self.generate_files()
self.generate_files(samples=sample_names)

def delete_column(self, column_name):
"""Delete `column_name` from info file
Expand Down Expand Up @@ -803,6 +803,13 @@ def _common_extend_steps(self, md_template):
----------
md_template : DataFrame
The metadata template file contents indexed by sample ids

Returns
-------
list of str
The new samples being added
list of str
The new columns being added
"""
with qdb.sql_connection.TRN:
# Check if we are adding new samples
Expand Down Expand Up @@ -887,6 +894,8 @@ def _common_extend_steps(self, md_template):
# Execute all the steps
qdb.sql_connection.TRN.execute()

return new_samples, new_cols

@classmethod
def exists(cls, obj_id):
r"""Checks if already exists a MetadataTemplate for the provided object
Expand Down Expand Up @@ -1247,9 +1256,9 @@ def extend(self, md_template):
with qdb.sql_connection.TRN:
md_template = self._clean_validate_template(
md_template, self.study_id, current_columns=self.categories())
self._common_extend_steps(md_template)
new_samples, new_columns = self._common_extend_steps(md_template)
self.validate(self.columns_restrictions)
self.generate_files()
self.generate_files(new_samples, new_columns)

def _update(self, md_template):
r"""Update values in the template
Expand All @@ -1259,6 +1268,13 @@ def _update(self, md_template):
md_template : DataFrame
The metadata template file contents indexed by samples ids

Returns
-------
set of str
The samples that were updated
set of str
The columns that were updated

Raises
------
QiitaDBError
Expand Down Expand Up @@ -1315,7 +1331,7 @@ def _update(self, md_template):
"There are no differences between the data stored in the "
"DB and the new data provided",
qdb.exceptions.QiitaDBWarning)
return
return None, None

changed.index.names = ['sample_name', 'column']
# the combination of np.where and boolean indexing produces
Expand Down Expand Up @@ -1344,7 +1360,9 @@ def _update(self, md_template):
# 4 XX.Sample3 physical_specimen_location new location
to_update.reset_index(inplace=True)
new_columns = []
samples_updated = []
for sid, df in to_update.groupby('sample_name'):
samples_updated.append(sid)
# getting just columns: column and to, and then using column
# as index will generate this for XX.Sample2:
# to
Expand All @@ -1363,9 +1381,9 @@ def _update(self, md_template):
self._table_name(self._id))
qdb.sql_connection.TRN.add(sql, [dumps(values), sid])

new_columns = list(set(new_columns).union(set(self.categories())))
nc = list(set(new_columns).union(set(self.categories())))
table_name = self._table_name(self.id)
values = dumps({"columns": new_columns})
values = dumps({"columns": nc})
sql = """UPDATE qiita.{0}
SET sample_values = %s
WHERE sample_id = '{1}'""".format(
Expand All @@ -1374,6 +1392,8 @@ def _update(self, md_template):

qdb.sql_connection.TRN.execute()

return set(samples_updated), set(new_columns)

def update(self, md_template):
r"""Update values in the template

Expand All @@ -1396,9 +1416,9 @@ def update(self, md_template):
# Clean and validate the metadata template given
new_map = self._clean_validate_template(
md_template, self.study_id, current_columns=self.categories())
self._update(new_map)
samples, columns = self._update(new_map)
self.validate(self.columns_restrictions)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Feels like it'd be more readable to test for not None or similar here, instead of having to go down into validate(), but I defer.

self.generate_files()
self.generate_files(samples, columns)

def extend_and_update(self, md_template):
"""Performs the update and extend operations at once
Expand All @@ -1416,10 +1436,19 @@ def extend_and_update(self, md_template):
with qdb.sql_connection.TRN:
md_template = self._clean_validate_template(
md_template, self.study_id, current_columns=self.categories())
self._common_extend_steps(md_template)
self._update(md_template)
new_samples, new_columns = self._common_extend_steps(md_template)
samples, columns = self._update(md_template)
if samples is None:
samples = new_samples
elif new_samples is not None:
samples.update(new_samples)
if columns is None:
columns = new_columns
elif new_columns is not None:
columns.update(new_columns)

self.validate(self.columns_restrictions)
self.generate_files()
self.generate_files(samples, columns)

def update_category(self, category, samples_and_values):
"""Update an existing column
Expand Down
9 changes: 8 additions & 1 deletion qiita_db/metadata_template/prep_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,8 +480,15 @@ def study_id(self):
qdb.sql_connection.TRN.add(sql, [self.id])
return qdb.sql_connection.TRN.execute_fetchlast()

def generate_files(self):
def generate_files(self, samples=None, columns=None):
r"""Generates all the files that contain data from this template

Parameters
----------
samples : iterable of str, optional
The samples that were added/updated
columns : iterable of str, optional
The columns that were added/updated
"""
with qdb.sql_connection.TRN:
# figuring out the filepath of the prep template
Expand Down
18 changes: 16 additions & 2 deletions qiita_db/metadata_template/sample_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,15 @@ def can_be_extended(self, new_samples, new_columns):
"""
return True, ""

def generate_files(self):
def generate_files(self, samples=None, columns=None):
r"""Generates all the files that contain data from this template

Parameters
----------
samples : iterable of str, optional
The samples that were added/updated
columns : iterable of str, optional
The columns that were added/updated
"""
with qdb.sql_connection.TRN:
# figuring out the filepath of the sample template
Expand All @@ -265,7 +272,14 @@ def generate_files(self):

# generating all new QIIME mapping files
for pt in qdb.study.Study(self._id).prep_templates():
pt.generate_files()
if samples is not None and samples and (
columns is None or not columns):
overlapping = set(samples) & set(pt.keys())
# if the prep has no overlapping sample ids, we can skip
# generationg the prep
if not overlapping:
continue
pt.generate_files(samples, columns)

@property
def ebi_sample_accessions(self):
Expand Down