Analysis refactor sync with master - DEPENDS ON #2138

qiita_core/tests/test_util.py

@@ -12,7 +12,8 @@
 
 from qiita_core.util import (
     send_email, qiita_test_checker, execute_as_transaction, get_qiita_version,
-    is_test_environment)
+    is_test_environment, get_release_info)
+from qiita_db.meta_util import generate_biom_and_metadata_release
 import qiita_db as qdb
 
 
@@ -64,6 +65,21 @@ def test_get_qiita_version(self):
         # testing just the version
         self.assertEqual(exp_version, qdb.__version__)
 
+    def test_get_release_info(self):
+        # making sure there is a release
+        generate_biom_and_metadata_release('private')
+        # just checking that is not empty cause the MD5 will change on every
+        # run
+        md5sum, filepath, timestamp = get_release_info('private')
+        self.assertNotEqual(md5sum, '')
+        self.assertNotEqual(filepath, '')
+        self.assertNotEqual(timestamp, '')
+
+        md5sum, filepath, timestamp = get_release_info('public')
+        self.assertEqual(md5sum, '')
+        self.assertEqual(filepath, '')
+        self.assertEqual(timestamp, '')
+
 
 if __name__ == '__main__':
     main()

qiita_core/util.py

@@ -11,6 +11,7 @@
 from os.path import dirname
 from git import Repo
 from git.exc import InvalidGitRepositoryError
+from moi import r_client
 
 from qiita_core.qiita_settings import qiita_config
 from qiita_pet import __version__ as qiita_pet_lib_version
@@ -141,3 +142,32 @@ def get_qiita_version():
         sha = ''
 
     return (qiita_pet_lib_version, sha)
+
+
+def get_release_info(study_status='public'):
+    """Returns the study status release MD5
+
+    Parameters
+    ----------
+    study_status : str, optional
+        The study status to search for. Note that this should always be set
+        to 'public' but having this exposed helps with testing. The other
+        options are 'private' and 'sandbox'
+
+    Returns
+    ------
+    str, str, str
+        The release MD5, filepath and timestamp
+    """
+    portal = qiita_config.portal
+    md5sum = r_client.get('%s:release:%s:md5sum' % (portal, study_status))
+    filepath = r_client.get('%s:release:%s:filepath' % (portal, study_status))
+    timestamp = r_client.get('%s:release:%s:time' % (portal, study_status))
+    if md5sum is None:
+        md5sum = ''
+    if filepath is None:
+        filepath = ''
+    if timestamp is None:
+        timestamp = ''
+
+    return md5sum, filepath, timestamp

qiita_db/artifact.py

@@ -274,9 +274,11 @@ def create(cls, filepaths, artifact_type, name=None, prep_template=None,
 
         Notes
         -----
-        The visibility of the artifact is set by default to `sandbox`
-        The timestamp of the artifact is set by default to `datetime.now()`
-        The value of `submitted_to_vamps` is set by default to `False`
+        The visibility of the artifact is set by default to `sandbox` if
+        prep_template is passed but if parents is passed we will inherit the
+        most closed visibility.
+        The timestamp of the artifact is set by default to `datetime.now()`.
+        The value of `submitted_to_vamps` is set by default to `False`.
         """
         # We need at least one file
         if not filepaths:
@@ -689,19 +691,21 @@ def visibility(self, value):
         only applies when the new visibility is more open than before.
         """
         with qdb.sql_connection.TRN:
+            # In order to correctly propagate the visibility we need to find
+            # the root of this artifact and then propagate to all the artifacts
+            sql = "SELECT * FROM qiita.find_artifact_roots(%s)"
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            root_id = qdb.sql_connection.TRN.execute_fetchlast()
+            root = qdb.artifact.Artifact(root_id)
+            # these are the ids of all the children from the root
+            ids = [a.id for a in root.descendants.nodes()]
+
             sql = """UPDATE qiita.artifact
                      SET visibility_id = %s
-                     WHERE artifact_id = %s"""
-            qdb.sql_connection.TRN.add(
-                sql, [qdb.util.convert_to_id(value, "visibility"), self.id])
+                     WHERE artifact_id IN %s"""
+            vis_id = qdb.util.convert_to_id(value, "visibility")
+            qdb.sql_connection.TRN.add(sql, [vis_id, tuple(ids)])
             qdb.sql_connection.TRN.execute()
-            # In order to correctly propagate the visibility upstream, we need
-            # to go one step at a time. By setting up the visibility of our
-            # parents first, we accomplish that, since they will propagate
-            # the changes to its parents
-            for p in self.parents:
-                visibilites = [[d.visibility] for d in p.descendants.nodes()]
-                p.visibility = qdb.util.infer_status(visibilites)
 
     @property
     def artifact_type(self):

qiita_db/meta_util.py

@@ -25,7 +25,8 @@
 from __future__ import division
 
 from moi import r_client
-from os import stat
+from os import stat, makedirs, rename
+from os.path import join, relpath, exists
 from time import strftime, localtime
 import matplotlib.pyplot as plt
 import matplotlib as mpl
@@ -34,8 +35,11 @@
 from StringIO import StringIO
 from future.utils import viewitems
 from datetime import datetime
+from tarfile import open as topen, TarInfo
+from hashlib import md5
 
 from qiita_core.qiita_settings import qiita_config
+from qiita_core.configuration_manager import ConfigurationManager
 import qiita_db as qdb
 
 
@@ -126,14 +130,21 @@ def validate_filepath_access_by_user(user, filepath_id):
             # the prep access is given by it's artifacts, if the user has
             # access to any artifact, it should have access to the prep
             # [0] cause we should only have 1
-            a = qdb.metadata_template.prep_template.PrepTemplate(
-                pid[0]).artifact
-            if (a.visibility == 'public' or a.study.has_access(user)):
-                return True
+            pt = qdb.metadata_template.prep_template.PrepTemplate(
+                pid[0])
+            a = pt.artifact
+            # however, the prep info file could not have any artifacts attached
+            # , in that case we will use the study access level
+            if a is None:
+                return qdb.study.Study(pt.study_id).has_access(user)
             else:
-                for c in a.descendants.nodes():
-                    if (c.visibility == 'public' or c.study.has_access(user)):
-                        return True
+                if (a.visibility == 'public' or a.study.has_access(user)):
+                    return True
+                else:
+                    for c in a.descendants.nodes():
+                        if ((c.visibility == 'public' or
+                             c.study.has_access(user))):
+                            return True
             return False
         # analyses
         elif anid:
@@ -305,7 +316,8 @@ def get_lat_longs():
                  WHERE table_name SIMILAR TO 'sample_[0-9]+'
                     AND table_schema = 'qiita'
                     AND column_name IN ('latitude', 'longitude')
-                    AND SPLIT_PART(table_name, '_', 2)::int IN %s;"""
+                    AND SPLIT_PART(table_name, '_', 2)::int IN %s
+                    GROUP BY table_name HAVING COUNT(column_name) = 2;"""
         qdb.sql_connection.TRN.add(sql, [tuple(portal_table_ids)])
 
         sql = [('SELECT CAST(latitude AS FLOAT), '
@@ -319,3 +331,99 @@ def get_lat_longs():
         qdb.sql_connection.TRN.add(sql)
 
         return qdb.sql_connection.TRN.execute_fetchindex()
+
+
+def generate_biom_and_metadata_release(study_status='public'):
+    """Generate a list of biom/meatadata filepaths and a tgz of those files
+
+    Parameters
+    ----------
+    study_status : str, optional
+        The study status to search for. Note that this should always be set
+        to 'public' but having this exposed helps with testing. The other
+        options are 'private' and 'sandbox'
+    """
+    studies = qdb.study.Study.get_by_status(study_status)
+    qiita_config = ConfigurationManager()
+    working_dir = qiita_config.working_dir
+    portal = qiita_config.portal
+    bdir = qdb.util.get_db_files_base_dir()
+    time = datetime.now().strftime('%m-%d-%y %H:%M:%S')
+
+    data = []
+    for s in studies:
+        # [0] latest is first, [1] only getting the filepath
+        sample_fp = relpath(s.sample_template.get_filepaths()[0][1], bdir)
+
+        for a in s.artifacts(artifact_type='BIOM'):
+            if a.processing_parameters is None:
+                continue
+
+            cmd_name = a.processing_parameters.command.name
+
+            # this loop is necessary as in theory an artifact can be
+            # generated from multiple prep info files
+            human_cmd = []
+            for p in a.parents:
+                pp = p.processing_parameters
+                pp_cmd_name = pp.command.name
+                if pp_cmd_name == 'Trimming':
+                    human_cmd.append('%s @ %s' % (
+                        cmd_name, str(pp.values['length'])))
+                else:
+                    human_cmd.append('%s, %s' % (cmd_name, pp_cmd_name))
+            human_cmd = ', '.join(human_cmd)
+
+            for _, fp, fp_type in a.filepaths:
+                if fp_type != 'biom' or 'only-16s' in fp:
+                    continue
+                fp = relpath(fp, bdir)
+                # format: (biom_fp, sample_fp, prep_fp, qiita_artifact_id,
+                #          human readable name)
+                for pt in a.prep_templates:
+                    for _, prep_fp in pt.get_filepaths():
+                        if 'qiime' not in prep_fp:
+                            break
+                    prep_fp = relpath(prep_fp, bdir)
+                    data.append((fp, sample_fp, prep_fp, a.id, human_cmd))
+
+    # writing text and tgz file
+    ts = datetime.now().strftime('%m%d%y-%H%M%S')
+    tgz_dir = join(working_dir, 'releases')
+    if not exists(tgz_dir):
+        makedirs(tgz_dir)
+    tgz_name = join(tgz_dir, '%s-%s-building.tgz' % (portal, study_status))
+    tgz_name_final = join(tgz_dir, '%s-%s.tgz' % (portal, study_status))
+    txt_hd = StringIO()
+    with topen(tgz_name, "w|gz") as tgz:
+        # writing header for txt
+        txt_hd.write(
+            "biom_fp\tsample_fp\tprep_fp\tqiita_artifact_id\tcommand\n")
+        for biom_fp, sample_fp, prep_fp, artifact_id, human_cmd in data:
+            txt_hd.write("%s\t%s\t%s\t%s\t%s\n" % (
+                biom_fp, sample_fp, prep_fp, artifact_id, human_cmd))
+            tgz.add(join(bdir, biom_fp), arcname=biom_fp, recursive=False)
+            tgz.add(join(bdir, sample_fp), arcname=sample_fp, recursive=False)
+            tgz.add(join(bdir, prep_fp), arcname=prep_fp, recursive=False)
+
+        txt_hd.seek(0)
+        info = TarInfo(name='%s-%s-%s.txt' % (portal, study_status, ts))
+        info.size = len(txt_hd.buf)
+        tgz.addfile(tarinfo=info, fileobj=txt_hd)
+
+    with open(tgz_name, "rb") as f:
+        md5sum = md5()
+        for c in iter(lambda: f.read(4096), b""):
+            md5sum.update(c)
+
+    rename(tgz_name, tgz_name_final)
+
+    vals = [
+        ('filepath', tgz_name_final[len(working_dir):], r_client.set),
+        ('md5sum', md5sum.hexdigest(), r_client.set),
+        ('time', time, r_client.set)]
+    for k, v, f in vals:
+        redis_key = '%s:release:%s:%s' % (portal, study_status, k)
+        # important to "flush" variables to avoid errors
+        r_client.delete(redis_key)
+        f(redis_key, v)