adding creation and modification timestamps to prep_templates (#3066)

* adding creation and modification timestamps to prep_templates

* Apply suggestions from code review [skip ci]

Co-authored-by: Yoshiki Vázquez Baeza <yoshiki@ucsd.edu>

Co-authored-by: Yoshiki Vázquez Baeza <yoshiki@ucsd.edu>

Author: antgonza

qiita_db/environment_manager.py

@@ -431,6 +431,7 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
         # for the test Study (1) so a lot of the tests actually expect this.
         # Now, trying to regenerate directly in the populate_test_db might
         # require too many dev hours so the easiest is just do it here
-        # UPDATE 02/27/19: moving to 74.sql as we added the file sizes
-        if test and sql_patch_filename == '74.sql':
+        # UPDATE 01/25/2021: moving to 81.sql as we added timestamps to
+        #                    prep info files
+        if test and sql_patch_filename == '81.sql':
             qdb.study.Study(1).sample_template.generate_files()

qiita_db/metadata_template/prep_template.py

@@ -7,7 +7,6 @@
 # -----------------------------------------------------------------------------
 from itertools import chain
 from os.path import join
-from time import strftime
 from copy import deepcopy
 from skbio.util import find_duplicates
 
@@ -507,11 +506,16 @@ def generate_files(self, samples=None, columns=None):
         with qdb.sql_connection.TRN:
             # figuring out the filepath of the prep template
             _id, fp = qdb.util.get_mountpoint('templates')[0]
+            # update timestamp in the DB first
+            qdb.sql_connection.TRN.add(
+                """UPDATE qiita.prep_template
+                   SET modification_timestamp = CURRENT_TIMESTAMP
+                   WHERE prep_template_id = %s""", [self._id])
+            ctime = self.modification_timestamp
             fp = join(fp, '%d_prep_%d_%s.txt' % (self.study_id, self._id,
-                      strftime("%Y%m%d-%H%M%S")))
+                      ctime.strftime("%Y%m%d-%H%M%S")))
             # storing the template
             self.to_file(fp)
-
             # adding the fp to the object
             fp_id = qdb.util.convert_to_id("prep_template", "filepath_type")
             self.add_filepath(fp, fp_id=fp_id)
@@ -666,3 +670,35 @@ def to_dataframe(self, add_ebi_accessions=False):
                 lambda sid: accessions[sid])
 
         return df
+
+    @property
+    def creation_timestamp(self):
+        """The creation timestamp of the prep information
+
+        Returns
+        -------
+        datetime.datetime
+            The creation timestamp of the prep information
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT creation_timestamp
+                     FROM qiita.prep_template
+                     WHERE prep_template_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchlast()
+
+    @property
+    def modification_timestamp(self):
+        """The modification timestamp of the prep information
+
+        Returns
+        -------
+        datetime.datetime
+            The modification timestamp of the prep information
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT modification_timestamp
+                     FROM qiita.prep_template
+                     WHERE prep_template_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchlast()

qiita_db/metadata_template/test/test_prep_template.py

@@ -11,6 +11,7 @@
 from os.path import join, exists
 from collections import Iterable
 from copy import deepcopy
+from datetime import datetime
 
 import numpy.testing as npt
 import pandas as pd
@@ -978,7 +979,14 @@ def test_validate_restrictions(self):
         self.assertFalse(success)
 
         metadata['target_gene'] = '16S rRNA'
+        # as we are testing the update functionality of a prep info file, we
+        # can also test that the timestamps are working correctly
+        current_ct = pt.creation_timestamp
+        current_mt = pt.modification_timestamp
+        self.assertTrue(current_ct < current_mt)
         pt.update(metadata)
+        self.assertEqual(current_ct, pt.creation_timestamp)
+        self.assertTrue(current_mt < pt.modification_timestamp)
         success, message = pt.validate_restrictions()
         success, message = pt.validate_restrictions()
         self.assertEqual(message, '')
@@ -994,6 +1002,12 @@ def test_create(self):
             self.metadata, self.test_study, self.data_type,
             name='New Prep For Test')
         self._common_creation_checks(pt, fp_count, "New Prep For Test")
+        # checking that the creation and modification timestamps are within
+        # 2 seconds of current time
+        dsecs = (datetime.now() - pt.modification_timestamp).total_seconds()
+        self.assertTrue(dsecs < 2)
+        dsecs = (datetime.now() - pt.creation_timestamp).total_seconds()
+        self.assertTrue(dsecs < 2)
         # cleaning
         qdb.metadata_template.prep_template.PrepTemplate.delete(pt.id)
 

qiita_db/support_files/patches/81.sql

@@ -0,0 +1,5 @@
+-- Jan 25, 2021
+-- Add creation_timestamp and modification_timestamp for qiita.prep_template
+
+ALTER TABLE qiita.prep_template ADD creation_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP;
+ALTER TABLE qiita.prep_template ADD modification_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP;

qiita_db/support_files/patches/python_patches/81.py

@@ -0,0 +1,25 @@
+from os.path import basename
+
+from qiita_db.sql_connection import TRN
+from qiita_db.study import Study
+
+
+for study in Study.iter():
+    for pt in study.prep_templates():
+        filepaths = pt.get_filepaths()
+        if filepaths:
+            # filepaths are returned in order so we can take the
+            # oldest and newest; then we get the filename and parse the
+            # creation time. Note that the filename comes in one of these
+            # formats: 1_prep_1_qiime_19700101-000000.txt or
+            # 1_prep_1_19700101-000000.txt
+            oldest = basename(filepaths[-1][1])[-19:-4].replace('-', ' ')
+            newest = basename(filepaths[0][1])[-19:-4].replace('-', ' ')
+
+            with TRN:
+                sql = """UPDATE qiita.prep_template
+                         SET creation_timestamp = %s,
+                             modification_timestamp = %s
+                         WHERE prep_template_id = %s"""
+                TRN.add(sql, [oldest, newest, pt.id])
+                TRN.execute()

qiita_db/support_files/qiita-db.dbs

@@ -939,6 +939,12 @@
 			<column name="deprecated" type="boolean" jt="-7" >
 				<defo><![CDATA[FALSE]]></defo>
 			</column>
+			<column name="creation_timestamp" type="timestamp" jt="-9" >
+				<defo><![CDATA[FALSE]]></defo>
+			</column>
+			<column name="modification_timestamp" type="timestamp" jt="-11" >
+				<defo><![CDATA[FALSE]]></defo>
+			</column>
 			<index name="pk_prep_template" unique="PRIMARY_KEY" >
 				<column name="prep_template_id" />
 			</index>
@@ -1889,4 +1895,4 @@ ALTER TABLE oauth_software ADD CONSTRAINT fk_oauth_software FOREIGN KEY ( client
 ]]></string>
 		</script>
 	</layout>
-</project>
+</project>