Info files from str to jsonb

qiita_db/environment_manager.py

@@ -382,6 +382,10 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
     Pulls the current patch from the settings table and applies all subsequent
     patches found in the patches directory.
     """
+    # we are going to open and close 2 main transactions; this is a required
+    # change since patch 68.sql where we transition to jsonb for all info
+    # files. The 2 main transitions are: (1) get the current settings,
+    # (2) each patch in their independent transaction
     with qdb.sql_connection.TRN:
         qdb.sql_connection.TRN.add("SELECT current_patch FROM settings")
         current_patch = qdb.sql_connection.TRN.execute_fetchlast()
@@ -398,19 +402,23 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
         else:
             next_patch_index = sql_patch_files.index(current_sql_patch_fp) + 1
 
-        patch_update_sql = "UPDATE settings SET current_patch = %s"
+    patch_update_sql = "UPDATE settings SET current_patch = %s"
 
-        for sql_patch_fp in sql_patch_files[next_patch_index:]:
-            sql_patch_filename = basename(sql_patch_fp)
+    for sql_patch_fp in sql_patch_files[next_patch_index:]:
+        sql_patch_filename = basename(sql_patch_fp)
 
-            # patch 43.sql is when we started testing patches
-            if sql_patch_filename == '43.sql' and test:
-                _populate_test_db()
+        py_patch_fp = corresponding_py_patch(
+            splitext(basename(sql_patch_fp))[0] + '.py')
+        py_patch_filename = basename(py_patch_fp)
 
-            py_patch_fp = corresponding_py_patch(
-                splitext(basename(sql_patch_fp))[0] + '.py')
-            py_patch_filename = basename(py_patch_fp)
+        # patch 43.sql is when we started testing patches, then in patch
+        # 68.sql is when we transitioned to jsonb for the info files; let's do
+        # this in its own transition
+        if sql_patch_filename == '68.sql' and test:
+            with qdb.sql_connection.TRN:
+                _populate_test_db()
 
+        with qdb.sql_connection.TRN:
             with open(sql_patch_fp, 'U') as patch_file:
                 if verbose:
                     print('\tApplying patch %s...' % sql_patch_filename)
@@ -425,3 +433,11 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
                     print('\t\tApplying python patch %s...'
                           % py_patch_filename)
                 execfile(py_patch_fp, {})
+
+        # before moving to jsonb for sample/prep info files (patch 69.sql),
+        # one of the patches used to regenerate the sample information file
+        # for the test Study (1) so alot of the tests actually expect this.
+        # Now, trying to regenerate directly in the populate_test_db might
+        # require too many dev hours so the easiest is just do it here
+        if test and sql_patch_filename == '69.sql':
+            qdb.study.Study(1).sample_template.generate_files()

qiita_db/meta_util.py

@@ -316,22 +316,25 @@ def get_lat_longs():
         sql = """SELECT DISTINCT table_name
                  FROM information_schema.columns
                  WHERE table_name SIMILAR TO 'sample_[0-9]+'
-                    AND table_schema = 'qiita'
-                    AND column_name IN ('latitude', 'longitude')
                     AND SPLIT_PART(table_name, '_', 2)::int IN %s
-                    GROUP BY table_name HAVING COUNT(column_name) = 2;"""
+                    AND table_schema = 'qiita'"""
         qdb.sql_connection.TRN.add(sql, [tuple(portal_table_ids)])
 
-        sql = [('SELECT CAST(latitude AS FLOAT), '
-                '       CAST(longitude AS FLOAT) '
-                'FROM qiita.%s '
-                'WHERE isnumeric(latitude) AND isnumeric(longitude) '
-                "AND latitude <> 'NaN' "
-                "AND longitude <> 'NaN' " % s)
+        # we are going to create multiple union selects to retrieve the
+        # latigute and longitude of all available studies. Note that UNION in
+        # PostgreSQL automatically removes duplicates
+        sql_query = """
+            SELECT CAST(sample_values->>'latitude' AS FLOAT),
+                   CAST(sample_values->>'longitude' AS FLOAT)
+            FROM qiita.%s
+            WHERE isnumeric(sample_values->>'latitude') AND
+                  isnumeric(sample_values->>'longitude')"""
+        sql = [sql_query % s
                for s in qdb.sql_connection.TRN.execute_fetchflatten()]
         sql = ' UNION '.join(sql)
         qdb.sql_connection.TRN.add(sql)
 
+        # note that we are returning set to remove duplicates
         return qdb.sql_connection.TRN.execute_fetchindex()