fix #814, fix #1636

Author: antgonza

.travis.yml

@@ -40,9 +40,10 @@ script:
   - qiita-env start_cluster qiita-general
   - qiita-env make --no-load-ontologies
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi
+  - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi
   - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi
   - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage -v --cover-package=qiita_db,qiita_pet,qiita_core,qiita_ware; fi
-  - flake8 qiita_* setup.py scripts/qiita scripts/qiita-env scripts/qiita-test-install
+  - flake8 qiita_* setup.py scripts/*
   - ls -R /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_pet/support_files/doc/
   - qiita pet webserver
 addons:

qiita_db/meta_util.py

@@ -25,6 +25,16 @@
 # -----------------------------------------------------------------------------
 from __future__ import division
 
+from moi import r_client
+from os import stat
+from time import strftime, localtime
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+from base64 import b64encode
+from urllib import quote
+from StringIO import StringIO
+from future.utils import viewitems
+
 from qiita_core.qiita_settings import qiita_config
 import qiita_db as qdb
 
@@ -122,6 +132,147 @@ def get_accessible_filepath_ids(user):
         return filepath_ids
 
 
+def update_redis_stats():
+    """Generate the system stats and save them in redis
+
+    Returns
+    -------
+    list of str
+        artifact filepaths that are not present in the file system
+    """
+    STUDY = qdb.study.Study
+    studies = {'public': STUDY.get_by_status('private'),
+               'private': STUDY.get_by_status('public'),
+               'sanbox': STUDY.get_by_status('sandbox')}
+    number_studies = {k: len(v) for k, v in viewitems(studies)}
+
+    number_of_samples = {}
+    ebi_samples = {}
+    for k, sts in viewitems(studies):
+        number_of_samples[k] = 0
+        for s in sts:
+            st = s.sample_template
+            if st is not None:
+                number_of_samples[k] += len(list(st.keys()))
+
+            ebi_samples_count = 0
+            for pt in s.prep_templates():
+                ebi_samples_count += len([
+                    1 for _, v in viewitems(pt.ebi_experiment_accessions)
+                    if v is not None and v != ''])
+            ebi_samples[s.id] = ebi_samples_count
+
+    num_users = qdb.util.get_count('qiita.qiita_user')
+
+    lat_longs = get_lat_longs()
+
+    num_studies_ebi = len(ebi_samples)
+    num_samples_ebi = sum([v for _, v in viewitems(ebi_samples)])
+
+    # generating file size stats
+    stats = []
+    missing_files = []
+    for k, sts in viewitems(studies):
+        for s in sts:
+            for a in s.artifacts():
+                for _, fp, dt in a.filepaths:
+                    try:
+                        s = stat(fp)
+                        stats.append((dt, s.st_size, strftime('%Y-%m',
+                                      localtime(s.st_ctime))))
+                    except:
+                        missing_files.append(fp)
+
+    summary = {}
+    all_dates = []
+    for ft, size, ym in stats:
+        if ft not in summary:
+            summary[ft] = {}
+        if ym not in summary[ft]:
+            summary[ft][ym] = 0
+            all_dates.append(ym)
+        summary[ft][ym] += size
+    all_dates = sorted(set(all_dates))
+
+    # sorting summaries
+    rm_from_data = ['html_summary', 'tgz', 'directory', 'raw_fasta', 'log',
+                    'biom', 'raw_sff', 'raw_qual']
+    ordered_summary = {}
+    for dt in summary:
+        if dt in rm_from_data:
+            continue
+        new_list = []
+        current_value = 0
+        for ad in all_dates:
+            if ad in summary[dt]:
+                current_value += summary[dt][ad]
+            new_list.append(current_value)
+        ordered_summary[dt] = new_list
+
+    plot_order = sorted([(k, ordered_summary[k][-1]) for k in ordered_summary],
+                        key=lambda x: x[1])
+
+    # helper function to generate y axis, modified from:
+    # http://stackoverflow.com/a/1094933
+    def sizeof_fmt(value, position):
+        number = None
+        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+            if abs(value) < 1024.0:
+                number = "%3.1f%s" % (value, unit)
+                break
+            value /= 1024.0
+        if number is None:
+            number = "%.1f%s" % (value, 'Yi')
+        return number
+
+    all_dates_axis = range(len(all_dates))
+    plt.locator_params(axis='y', nbins=10)
+    plt.figure(figsize=(20, 10))
+    for k, v in plot_order:
+        plt.plot(all_dates_axis, ordered_summary[k], linewidth=2, label=k)
+
+    plt.xticks(all_dates_axis, all_dates)
+    plt.legend()
+    plt.grid()
+    ax = plt.gca()
+    ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(sizeof_fmt))
+    plt.xlabel('Date')
+    plt.ylabel('Storage space per data type')
+
+    plot = StringIO()
+    plt.savefig(plot, format='png')
+    plot.seek(0)
+    img = '<img src = "%s"/>' % (
+        'data:image/png;base64,' + quote(b64encode(plot.buf)))
+
+    portal = qiita_config.portal
+    keys = [
+        'number_studies', 'number_of_samples', 'num_users', 'lat_longs',
+        'num_studies_ebi', 'num_samples_ebi', 'img']
+    for k in keys:
+        redis_key = '%s:stats:%s' % (portal, k)
+
+        # storing dicts
+        if k == 'number_studies':
+            r_client.hmset(redis_key, number_studies)
+        elif k == 'number_of_samples':
+            r_client.hmset(redis_key, number_of_samples)
+        # single values
+        elif k == 'num_users':
+            r_client.set(redis_key, num_users)
+        elif k == 'num_studies_ebi':
+            r_client.set(redis_key, num_studies_ebi)
+        elif k == 'num_samples_ebi':
+            r_client.set(redis_key, num_samples_ebi)
+        elif k == 'img':
+            r_client.set(redis_key, img)
+        # storing tuples
+        elif k == 'lat_longs':
+            r_client.set(redis_key, lat_longs)
+
+    return missing_files
+
+
 def get_lat_longs():
     """Retrieve the latitude and longitude of all the samples in the DB
 

qiita_db/test/test_meta_util.py

@@ -10,6 +10,7 @@
 
 import pandas as pd
 
+from moi import r_client
 from qiita_core.qiita_settings import qiita_config
 from qiita_core.util import qiita_test_checker
 
@@ -180,6 +181,59 @@ def test_get_lat_longs_EMP_portal(self):
 
         self.assertItemsEqual(obs, exp)
 
+    def test_update_redis_stats(self):
+        qdb.meta_util.update_redis_stats()
+
+        # checking values from redis
+        portal = qiita_config.portal
+        keys = [
+            'number_studies', 'number_of_samples', 'num_users', 'lat_longs',
+            'num_studies_ebi', 'num_samples_ebi', 'img']
+
+        for k in keys:
+            redis_key = '%s:stats:%s' % (portal, k)
+            # retrieving dicts
+            if k == 'number_studies':
+                data = r_client.hgetall(redis_key)
+                self.assertEqual(data, {'sanbox': '2', 'public': '0',
+                                        'private': '1'})
+            elif k == 'number_of_samples':
+                data = r_client.hgetall(redis_key)
+                self.assertEqual(data, {'sanbox': '1', 'public': '0',
+                                        'private': '27'})
+            # single values
+            elif k == 'num_users':
+                data = r_client.get(redis_key)
+                self.assertEqual(data, '4')
+            elif k == 'num_studies_ebi':
+                data = r_client.get(redis_key)
+                self.assertEqual(data, '3')
+            elif k == 'num_samples_ebi':
+                data = r_client.get(redis_key)
+                self.assertEqual(data, '54')
+            elif k == 'img':
+                # not testing image!
+                data = r_client.get(redis_key)
+            # storing tuples and single values
+            elif k == 'lat_longs':
+                data = r_client.get(redis_key)
+                self.assertEqual(data, EXP_LAT_LONG)
+
+
+EXP_LAT_LONG = (
+    '[[0.291867635913, 68.5945325743], [68.0991287718, 34.8360987059],'
+    ' [10.6655599093, 70.784770579], [40.8623799474, 6.66444220187],'
+    ' [13.089194595, 92.5274472082], [84.0030227585, 66.8954849864],'
+    ' [12.7065957714, 84.9722975792], [78.3634273709, 74.423907894],'
+    ' [82.8302905615, 86.3615778099], [53.5050692395, 31.6056761814],'
+    ' [43.9614715197, 82.8516734159], [29.1499460692, 82.1270418227],'
+    ' [23.1218032799, 42.838497795], [12.6245524972, 96.0693176066],'
+    ' [38.2627021402, 3.48274264219], [74.0894932572, 65.3283470202],'
+    ' [35.2374368957, 68.5041623253], [4.59216095574, 63.5115213108],'
+    ' [95.2060749748, 27.3592668624], [68.51099627, 2.35063674718],'
+    ' [85.4121476399, 15.6526750776], [60.1102854322, 74.7123248382],'
+    ' [3.21190859967, 26.8138925876], [57.571893782, 32.5563076447],'
+    ' [44.9725384282, 66.1920014699], [42.42, 41.41]]')
 
 if __name__ == '__main__':
     main()

qiita_db/test/test_util.py

@@ -8,7 +8,7 @@
 
 from unittest import TestCase, main
 from tempfile import mkstemp
-from os import close, remove
+from os import close, remove, mkdir
 from os.path import join, exists, basename
 from shutil import rmtree
 from datetime import datetime
@@ -365,6 +365,20 @@ def _common_purge_filpeaths_test(self):
     def test_purge_filepaths(self):
         self._common_purge_filpeaths_test()
 
+    def test_empty_trash_upload_folder(self):
+        # creating file to delete so we know it actually works
+        study_id = '1'
+        uploads_fp = join(qdb.util.get_mountpoint("uploads")[0][1], study_id)
+        trash = join(uploads_fp, 'trash')
+        if not exists(trash):
+            mkdir(trash)
+        fp = join(trash, 'my_file_to_delete.txt')
+        open(fp, 'w').close()
+
+        self.assertTrue(exists(fp))
+        qdb.util.empty_trash_upload_folder()
+        self.assertFalse(exists(fp))
+
     def test_purge_filepaths_null_cols(self):
         # For more details about the source of the issue that motivates this
         # test: http://www.depesz.com/2008/08/13/nulls-vs-not-in/

qiita_db/util.py

@@ -714,7 +714,17 @@ def path_builder(db_dir, filepath, mountpoint, subdirectory, obj_id):
                 for fpid, fp, fp_type_, m, s in results]
 
 
-def purge_filepaths():
+def _rm_files(TRN, fp):
+    # Remove the data
+    if exists(fp):
+        if isdir(fp):
+            func = rmtree
+        else:
+            func = remove
+        TRN.add_post_commit_func(func, fp)
+
+
+def purge_filepaths(delete_files=True):
     r"""Goes over the filepath table and remove all the filepaths that are not
     used in any place
     """
@@ -751,18 +761,39 @@ def purge_filepaths():
         sql = "DELETE FROM qiita.filepath WHERE filepath_id=%s"
         db_results = qdb.sql_connection.TRN.execute_fetchindex()
         for fp_id, fp, fp_type, dd_id in db_results:
-            qdb.sql_connection.TRN.add(sql, [fp_id])
+            if delete_files:
+                qdb.sql_connection.TRN.add(sql, [fp_id])
+                fp = join(get_mountpoint_path_by_id(dd_id), fp)
+                _rm_files(qdb.sql_connection.TRN, fp)
+            else:
+                print fp, fp_type
 
-            # Remove the data
-            fp = join(get_mountpoint_path_by_id(dd_id), fp)
-            if exists(fp):
-                if fp_type is 'directory':
-                    func = rmtree
-                else:
-                    func = remove
-                qdb.sql_connection.TRN.add_post_commit_func(func, fp)
+        if delete_files:
+            qdb.sql_connection.TRN.execute()
 
-        qdb.sql_connection.TRN.execute()
+
+def empty_trash_upload_folder(delete_files=True):
+    r"""Delete all files in the trash folder inside each of the upload
+    folders"""
+    gfp = partial(join, get_db_files_base_dir())
+    with qdb.sql_connection.TRN:
+        sql = """SELECT mountpoint
+                 FROM qiita.data_directory
+                 WHERE data_type = 'uploads'"""
+        qdb.sql_connection.TRN.add(sql)
+
+        for mp in qdb.sql_connection.TRN.execute_fetchflatten():
+            for path, dirs, files in walk(gfp(mp)):
+                if path.endswith('/trash'):
+                    if delete_files:
+                        for f in files:
+                            fp = join(path, f)
+                            _rm_files(qdb.sql_connection.TRN, fp)
+                    else:
+                        print files
+
+        if delete_files:
+            qdb.sql_connection.TRN.execute()
 
 
 def move_filepaths_to_upload_folder(study_id, filepaths):

scripts/qiita-cron-job

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from qiita_db.util import purge_filepaths, empty_trash_upload_folder
+from qiita_db.meta_util import update_redis_stats
+
+
+# This script will perform these jobs:
+# 1. purge_filepaths: remove files from that are leftover in the
+#    qiita.filepath and are present in the filesystem
+# 2. empty_trash_upload_folder: remove files that are present in the trash
+#    of the upload folders
+# 3. update_redis_stats: updates the redis stats information
+#
+# Note that is responsability of the Qiita install system admin to add to a
+# cron job this script and responsible to define how often it should run
+
+
+def call_purge_filepaths():
+    """remove files from that are leftover in the qiita.filepath and are
+    present in the filesystem"""
+    purge_filepaths(True)
+
+
+def call_empty_trash_upload_folder():
+    """updates the redis stats information"""
+    empty_trash_upload_folder(True)
+
+
+def call_update_redis_stats():
+    """updates the redis stats information"""
+    update_redis_stats()
+
+
+def main():
+    call_purge_filepaths()
+    call_empty_trash_upload_folder()
+    call_update_redis_stats()
+
+
+if __name__ == "__main__":
+    main()