diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py index 6326060d2..af0215886 100644 --- a/qiita_db/metadata_template/base_metadata_template.py +++ b/qiita_db/metadata_template/base_metadata_template.py @@ -696,7 +696,7 @@ def _common_delete_sample_steps(self, sample_names): qdb.sql_connection.TRN.add(sql2, [sn, self.id]) qdb.sql_connection.TRN.execute() - self.generate_files() + self.generate_files(samples=sample_names) def delete_column(self, column_name): """Delete `column_name` from info file @@ -803,6 +803,13 @@ def _common_extend_steps(self, md_template): ---------- md_template : DataFrame The metadata template file contents indexed by sample ids + + Returns + ------- + list of str + The new samples being added + list of str + The new columns being added """ with qdb.sql_connection.TRN: # Check if we are adding new samples @@ -887,6 +894,8 @@ def _common_extend_steps(self, md_template): # Execute all the steps qdb.sql_connection.TRN.execute() + return new_samples, new_cols + @classmethod def exists(cls, obj_id): r"""Checks if already exists a MetadataTemplate for the provided object @@ -1247,9 +1256,9 @@ def extend(self, md_template): with qdb.sql_connection.TRN: md_template = self._clean_validate_template( md_template, self.study_id, current_columns=self.categories()) - self._common_extend_steps(md_template) + new_samples, new_columns = self._common_extend_steps(md_template) self.validate(self.columns_restrictions) - self.generate_files() + self.generate_files(new_samples, new_columns) def _update(self, md_template): r"""Update values in the template @@ -1259,6 +1268,13 @@ def _update(self, md_template): md_template : DataFrame The metadata template file contents indexed by samples ids + Returns + ------- + set of str + The samples that were updated + set of str + The columns that were updated + Raises ------ QiitaDBError @@ -1315,7 +1331,7 @@ def _update(self, md_template): "There are no differences between the data stored in the " "DB and the new data provided", qdb.exceptions.QiitaDBWarning) - return + return None, None changed.index.names = ['sample_name', 'column'] # the combination of np.where and boolean indexing produces @@ -1344,7 +1360,9 @@ def _update(self, md_template): # 4 XX.Sample3 physical_specimen_location new location to_update.reset_index(inplace=True) new_columns = [] + samples_updated = [] for sid, df in to_update.groupby('sample_name'): + samples_updated.append(sid) # getting just columns: column and to, and then using column # as index will generate this for XX.Sample2: # to @@ -1363,9 +1381,9 @@ def _update(self, md_template): self._table_name(self._id)) qdb.sql_connection.TRN.add(sql, [dumps(values), sid]) - new_columns = list(set(new_columns).union(set(self.categories()))) + nc = list(set(new_columns).union(set(self.categories()))) table_name = self._table_name(self.id) - values = dumps({"columns": new_columns}) + values = dumps({"columns": nc}) sql = """UPDATE qiita.{0} SET sample_values = %s WHERE sample_id = '{1}'""".format( @@ -1374,6 +1392,8 @@ def _update(self, md_template): qdb.sql_connection.TRN.execute() + return set(samples_updated), set(new_columns) + def update(self, md_template): r"""Update values in the template @@ -1396,9 +1416,9 @@ def update(self, md_template): # Clean and validate the metadata template given new_map = self._clean_validate_template( md_template, self.study_id, current_columns=self.categories()) - self._update(new_map) + samples, columns = self._update(new_map) self.validate(self.columns_restrictions) - self.generate_files() + self.generate_files(samples, columns) def extend_and_update(self, md_template): """Performs the update and extend operations at once @@ -1416,10 +1436,19 @@ def extend_and_update(self, md_template): with qdb.sql_connection.TRN: md_template = self._clean_validate_template( md_template, self.study_id, current_columns=self.categories()) - self._common_extend_steps(md_template) - self._update(md_template) + new_samples, new_columns = self._common_extend_steps(md_template) + samples, columns = self._update(md_template) + if samples is None: + samples = new_samples + elif new_samples is not None: + samples.update(new_samples) + if columns is None: + columns = new_columns + elif new_columns is not None: + columns.update(new_columns) + self.validate(self.columns_restrictions) - self.generate_files() + self.generate_files(samples, columns) def update_category(self, category, samples_and_values): """Update an existing column diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index b93d192e1..fc110c2f6 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -480,8 +480,15 @@ def study_id(self): qdb.sql_connection.TRN.add(sql, [self.id]) return qdb.sql_connection.TRN.execute_fetchlast() - def generate_files(self): + def generate_files(self, samples=None, columns=None): r"""Generates all the files that contain data from this template + + Parameters + ---------- + samples : iterable of str, optional + The samples that were added/updated + columns : iterable of str, optional + The columns that were added/updated """ with qdb.sql_connection.TRN: # figuring out the filepath of the prep template diff --git a/qiita_db/metadata_template/sample_template.py b/qiita_db/metadata_template/sample_template.py index 71d5b74f0..d75c45068 100644 --- a/qiita_db/metadata_template/sample_template.py +++ b/qiita_db/metadata_template/sample_template.py @@ -249,8 +249,15 @@ def can_be_extended(self, new_samples, new_columns): """ return True, "" - def generate_files(self): + def generate_files(self, samples=None, columns=None): r"""Generates all the files that contain data from this template + + Parameters + ---------- + samples : iterable of str, optional + The samples that were added/updated + columns : iterable of str, optional + The columns that were added/updated """ with qdb.sql_connection.TRN: # figuring out the filepath of the sample template @@ -265,7 +272,14 @@ def generate_files(self): # generating all new QIIME mapping files for pt in qdb.study.Study(self._id).prep_templates(): - pt.generate_files() + if samples is not None and samples and ( + columns is None or not columns): + overlapping = set(samples) & set(pt.keys()) + # if the prep has no overlapping sample ids, we can skip + # generationg the prep + if not overlapping: + continue + pt.generate_files(samples, columns) @property def ebi_sample_accessions(self):