setting up redbiom

.travis.yml

@@ -1,16 +1,16 @@
 dist: precise
 language: python
-sudo: false
+sudo: required
 env:
   global:
     - PYTHON_VERSION=2.7
-  matrix:
-    - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_db
-    - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_pet
-    - TEST_ADD_STUDIES=False COVER_PACKAGE="qiita_core qiita_ware"
-    - TEST_ADD_STUDIES=True
+  # matrix:
+  #   - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_db
+  #   - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_pet
+  #   - TEST_ADD_STUDIES=False COVER_PACKAGE="qiita_core qiita_ware"
+  #   - TEST_ADD_STUDIES=True
 before_install:
-  - redis-server --version
+  - sudo -H -u redis redis-server /etc/redis/redis.conf --port 7777 &
   - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   - chmod +x miniconda.sh
   - ./miniconda.sh -b
@@ -28,19 +28,37 @@ install:
   # install a few of the dependencies that pip would otherwise try to install
   # when intalling scikit-bio
   - travis_retry conda create --yes -n qiita python=$PYTHON_VERSION pip nose flake8
-    pyzmq networkx pyparsing natsort mock future libgfortran seaborn
+    pyzmq networkx pyparsing natsort mock future libgfortran seaborn nltk
     'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1'
   - source activate qiita
   - pip install -U pip
   - pip install sphinx sphinx-bootstrap-theme 'ipython[all]==2.4.1' nose-timer codecov
-  - travis_retry pip install . --process-dependency-links
   - 'echo "backend: Agg" > matplotlibrc'
+  # installing redbiom, first webdis
+  - git clone https://github.com/nicolasff/webdis
+  - pushd webdis
+  - make
+  - ./webdis &
+  - popd
+  - travis_retry pip install . --process-dependency-links
   # Install the biom plugin so we can run the analysis tests
   - pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
   - pip install https://github.com/qiita-spots/qtp-biom/archive/master.zip --process-dependency-links
+  - pip install https://github.com/qiita-spots/qtp-target-gene/archive/master.zip --process-dependency-links
   - export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
+  # loading redbiom with Qiita's test set
+  # but first let's make sure redis is empty
+  # following https://github.com/biocore/redbiom/blob/master/Makefile
+  - export REDBIOM_HOST=http://127.0.0.1:7379
+  - curl -s http://127.0.0.1:7379/FLUSHALL > /dev/null
+  - redbiom admin scripts-writable
+  - redbiom admin create-context --name "qiita-test" --description "qiita-test context"
+  - redbiom admin load-sample-metadata --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt
+  - redbiom admin load-sample-metadata-search --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt
+  - redbiom admin load-sample-data --table `pwd`/qiita_db/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test
   - mkdir ~/.qiita_plugins
   - cp $PWD/qiita_core/support_files/BIOM\ type_2.1.4.conf ~/.qiita_plugins
+  - cp $PWD/qiita_core/support_files/Target\ Gene\ type_0.1.0.conf ~/.qiita_plugins
 before_script:
     # Some of the tests rely on the plugin system to complete successfuly.
     # Thus, we need a qiita webserver running to be able to execute the tests.
@@ -59,9 +77,11 @@ before_script:
     fi
 script:
   - sleep 5
+  - pip freeze
+  - qiita-test-install
+  - python qiita_pet/handlers/api_proxy/tests/test_artifact.py -vvvv
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi
-  - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi
   - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=$COVER_PACKAGE; fi
   - flake8 qiita_* setup.py scripts/*
   - ls -R /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_pet/support_files/doc/

INSTALL.md

@@ -60,6 +60,7 @@ Install the non-python dependencies
 
 * [PostgreSQL](http://www.postgresql.org/download/) (minimum required version 9.3.5, we have tested most extensively with 9.3.6)
 * [redis-server](http://redis.io) (we have tested most extensively with 2.8.17)
+* [webdis] (https://github.com/nicolasff/webdis) (latest version should be fine but we have tested the most with 9ee6fe2 - Feb 6, 2016)
 
 There are several options to install these dependencies depending on your needs:
 
@@ -87,6 +88,28 @@ brew update
 brew install homebrew/versions/redis28
 ```
 
+### webdis
+
+Note that this is the only package that assumes that Qiita is already installed (due to library dependencies). Also, that the general suggestion is to have 2 redis servers running, one for webdis/redbiom and the other for Qiita. The reason for multiple redis servers is so that the redbiom cache can be flushed without impacting the operation of the qiita server itself.
+
+The following instructions install, compile and pre-populates the redbiom redis DB so we assume that redis is running on the default port and that Qiita is fully installed as the redbiom package is installed with Qiita.
+
+```
+git clone https://github.com/nicolasff/webdis
+pushd webdis
+make
+./webdis &
+popd
+# note that this assumes that Qiita is already installed
+fp=`python -c 'import qiita_db; print qiita_db.__file__'`
+qdbd=`dirname $fp`
+redbiom admin create-context --name "qiita-test" --description "qiita-test context"
+redbiom admin load-sample-metadata --metadata ${qdbd}/support_files/test_data/templates/1_19700101-000000.txt
+redbiom admin load-sample-metadata-search --metadata ${qdbd}/support_files/test_data/templates/1_19700101-000000.txt
+redbiom admin load-observations --table ${qdbd}/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test
+redbiom admin load-sample-data --table ${qdbd}/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test
+```
+
 
 Install Qiita development version and its python dependencies
 -------------------------------------------------------------
@@ -163,6 +186,13 @@ Next, make a test environment:
 qiita-env make --no-load-ontologies
 ```
 
+Finally, redbiom relies on the REDBIOM_HOST environment variable to set the URL to query. By default is set to http://127.0.0.1:7379, which is the webdis default. For example you could:
+
+```bash
+export REDBIOM_HOST=http://my_host.com:7329
+```
+
+
 ## Start Qiita
 
 Start postgres (instructions vary depending on operating system and install method).

qiita_core/support_files/Target Gene type_0.1.0.conf

@@ -0,0 +1,13 @@
+[main]
+NAME = Target Gene type
+VERSION = 0.1.0
+DESCRIPTION = Target gene artifact types plugin
+ENVIRONMENT_SCRIPT = source activate qiita
+START_SCRIPT = start_target_gene_types
+PLUGIN_TYPE = artifact definition
+PUBLICATIONS =
+
+[oauth2]
+SERVER_CERT = /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_core/support_files/server.crt
+CLIENT_ID = 4MOBzUBHBtUmwhaC258H7PS0rBBLyGQrVxGPgc9g305bvVhf6h
+CLIENT_SECRET = rFb7jwAb3UmSUN57Bjlsi4DTl2owLwRpwCc0SggRNEVb2Ebae2p5Umnq20rNMhmqN

qiita_core/support_files/config_test.cfg

@@ -91,7 +91,7 @@ EMAIL = example@domain.com
 # ----------------------------- Redis settings --------------------------------
 [redis]
 HOST = localhost
-PORT = 6379
+PORT = 7777
 PASSWORD =
 # The redis database you will use, redis has a max of 16.
 # Qiita should have its own database

qiita_db/metadata_template/util.py

@@ -14,7 +14,6 @@
 import pandas as pd
 import numpy as np
 import warnings
-from skbio.io.util import open_file
 from skbio.util import find_duplicates
 
 import qiita_db as qdb
@@ -102,7 +101,7 @@ def load_template_to_dataframe(fn, index='sample_name'):
     """
     # Load in file lines
     holdfile = None
-    with open_file(fn, mode='U') as f:
+    with qdb.util.open_file(fn, mode='U') as f:
         errors = defaultdict(list)
         holdfile = f.readlines()
         # here we are checking for non UTF-8 chars
@@ -333,7 +332,7 @@ def looks_like_qiime_mapping_file(fp):
     some other different column.
     """
     first_line = None
-    with open_file(fp, mode='U') as f:
+    with qdb.util.open_file(fp, mode='U') as f:
         first_line = f.readline()
     if not first_line:
         return False

qiita_db/processing_job.py

@@ -754,6 +754,7 @@ def _set_error(self, error):
         qiita_db.exceptions.QiitaDBOperationNotPermittedError
             If the status of the job is 'success'
         """
+        print '\n\n\n-->>', error, '\n\n'
         with qdb.sql_connection.TRN:
             if self.status == 'success':
                 raise qdb.exceptions.QiitaDBOperationNotPermittedError(

qiita_db/search.py

@@ -69,6 +69,7 @@
 
 import pandas as pd
 from future.utils import viewitems
+from future.builtins import str
 
 from qiita_core.qiita_settings import qiita_config
 import qiita_db as qdb

qiita_db/test/test_util.py

@@ -7,13 +7,15 @@
 # -----------------------------------------------------------------------------
 
 from unittest import TestCase, main
-from tempfile import mkstemp
+from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile
 from os import close, remove, makedirs, mkdir
 from os.path import join, exists, basename
 from shutil import rmtree
 from datetime import datetime
 from functools import partial
 from string import punctuation
+import h5py
+from six import StringIO, BytesIO
 
 import pandas as pd
 
@@ -869,5 +871,77 @@ def test_get_artifacts_information(self):
         self.assertItemsEqual(obs, exp)
 
 
+class TestFilePathOpening(TestCase):
+    """Tests adapted from scikit-bio's skbio.io.util tests"""
+    def test_is_string_or_bytes(self):
+        self.assertTrue(qdb.util._is_string_or_bytes('foo'))
+        self.assertTrue(qdb.util._is_string_or_bytes(u'foo'))
+        self.assertTrue(qdb.util._is_string_or_bytes(b'foo'))
+        self.assertFalse(qdb.util._is_string_or_bytes(StringIO('bar')))
+        self.assertFalse(qdb.util._is_string_or_bytes([1]))
+
+    def test_file_closed(self):
+        """File gets closed in decorator"""
+        f = NamedTemporaryFile('r')
+        filepath = f.name
+        with qdb.util.open_file(filepath) as fh:
+            pass
+        self.assertTrue(fh.closed)
+
+    def test_file_closed_harder(self):
+        """File gets closed in decorator, even if exceptions happen."""
+        f = NamedTemporaryFile('r')
+        filepath = f.name
+        try:
+            with qdb.util.open_file(filepath) as fh:
+                raise TypeError
+        except TypeError:
+            self.assertTrue(fh.closed)
+        else:
+            # If we're here, no exceptions have been raised inside the
+            # try clause, so the context manager swallowed them. No
+            # good.
+            raise Exception("`open_file` didn't propagate exceptions")
+
+    def test_filehandle(self):
+        """Filehandles slip through untouched"""
+        with TemporaryFile('r') as fh:
+            with qdb.util.open_file(fh) as ffh:
+                self.assertTrue(fh is ffh)
+            # And it doesn't close the file-handle
+            self.assertFalse(fh.closed)
+
+    def test_StringIO(self):
+        """StringIO (useful e.g. for testing) slips through."""
+        f = StringIO("File contents")
+        with qdb.util.open_file(f) as fh:
+            self.assertTrue(fh is f)
+
+    def test_BytesIO(self):
+        """BytesIO (useful e.g. for testing) slips through."""
+        f = BytesIO(b"File contents")
+        with qdb.util.open_file(f) as fh:
+            self.assertTrue(fh is f)
+
+    def test_hdf5IO(self):
+        f = h5py.File('test', driver='core', backing_store=False)
+        with qdb.util.open_file(f) as fh:
+            self.assertTrue(fh is f)
+
+    def test_hdf5IO_open(self):
+        name = None
+        with NamedTemporaryFile(delete=False) as fh:
+            name = fh.name
+            fh.close()
+
+            h5file = h5py.File(name, 'w')
+            h5file.close()
+
+            with qdb.util.open_file(name) as fh_inner:
+                self.assertTrue(isinstance(fh_inner, h5py.File))
+
+        remove(name)
+
+
 if __name__ == '__main__':
     main()

qiita_db/util.py

@@ -54,6 +54,9 @@
 from json import dumps
 from datetime import datetime
 from itertools import chain
+from contextlib import contextmanager
+from future.builtins import bytes, str
+import h5py
 
 from qiita_core.exceptions import IncompetentQiitaDeveloperError
 import qiita_db as qdb
@@ -1481,3 +1484,67 @@ def get_artifacts_information(artifact_ids, only_biom=True):
                     'files': filepaths})
 
             return results
+
+
+def _is_string_or_bytes(s):
+    """Returns True if input argument is string (unicode or not) or bytes.
+    """
+    return isinstance(s, str) or isinstance(s, bytes)
+
+
+def _get_filehandle(filepath_or, *args, **kwargs):
+    """Open file if `filepath_or` looks like a string/unicode/bytes, else
+    pass through.
+    """
+    if _is_string_or_bytes(filepath_or):
+        if h5py.is_hdf5(filepath_or):
+            fh, own_fh = h5py.File(filepath_or, *args, **kwargs), True
+        else:
+            fh, own_fh = open(filepath_or, *args, **kwargs), True
+    else:
+        fh, own_fh = filepath_or, False
+    return fh, own_fh
+
+
+@contextmanager
+def open_file(filepath_or, *args, **kwargs):
+    """Context manager, like ``open``, but lets file handles and file like
+    objects pass untouched.
+
+    It is useful when implementing a function that can accept both
+    strings and file-like objects (like numpy.loadtxt, etc).
+
+    This method differs slightly from scikit-bio's implementation in that it
+    handles HDF5 files appropriately.
+
+    Parameters
+    ----------
+    filepath_or : str/bytes/unicode string or file-like
+         If string, file to be opened using ``h5py.File`` if the file is an
+         HDF5 file, otherwise builtin ``open`` will be used. If it is not a
+         string, the object is just returned untouched.
+
+    Other parameters
+    ----------------
+    args, kwargs : tuple, dict
+        When `filepath_or` is a string, any extra arguments are passed
+        on to the ``open`` builtin.
+
+    Examples
+    --------
+    >>> with open_file('filename') as f:  # doctest: +SKIP
+    ...     pass
+    >>> fh = open('filename')             # doctest: +SKIP
+    >>> with open_file(fh) as f:          # doctest: +SKIP
+    ...     pass
+    >>> fh.closed                         # doctest: +SKIP
+    False
+    >>> fh.close()                        # doctest: +SKIP
+
+    """
+    fh, own_fh = _get_filehandle(filepath_or, *args, **kwargs)
+    try:
+        yield fh
+    finally:
+        if own_fh:
+            fh.close()

qiita_pet/handlers/api_proxy/artifact.py

@@ -10,8 +10,8 @@
 from json import dumps
 
 from future.utils import viewitems
+from itertools import chain
 from moi import r_client
-from skbio.util import flatten
 
 from qiita_core.util import execute_as_transaction
 from qiita_core.qiita_settings import qiita_config
@@ -103,8 +103,8 @@ def artifact_get_prep_req(user_id, artifact_ids):
         if access_error:
             return access_error
 
-        samples[aid] = flatten(
-            [pt.keys() for pt in Artifact(aid).prep_templates])
+        samples[aid] = list(chain(
+            *[pt.keys() for pt in Artifact(aid).prep_templates]))
 
     return {'status': 'success', 'msg': '', 'data': samples}
 
@@ -183,6 +183,7 @@ def artifact_post_req(user_id, filepaths, artifact_type, name,
         uploads_path = get_mountpoint('uploads')[0][1]
         path_builder = partial(join, uploads_path, str(study_id))
         cleaned_filepaths = {}
+
         for ftype, file_list in viewitems(filepaths):
             # JavaScript sends us this list as a comma-separated list
             for fp in file_list.split(','):