Analysis refactor db (#2040)

.travis.yml

@@ -4,7 +4,9 @@ env:
   global:
     - PYTHON_VERSION=2.7
   matrix:
-    - TEST_ADD_STUDIES=False
+    - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_db
+    - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_pet
+    - TEST_ADD_STUDIES=False COVER_PACKAGE="qiita_core qiita_ware"
     - TEST_ADD_STUDIES=True
 before_install:
   - redis-server --version
@@ -18,7 +20,9 @@ before_install:
   - wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh
   - chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh
   - ./ascp-install-3.5.4.102989-linux-64-qiita.sh
-  - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then openssl aes-256-cbc -K $encrypted_a2e23aea5f14_key -iv $encrypted_a2e23aea5f14_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d ; fi
+  # once we have ebi testing we should uncomment this line
+  # look for EBI below as it's part of this
+  # - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then openssl aes-256-cbc -K $encrypted_a2e23aea5f14_key -iv $encrypted_a2e23aea5f14_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d ; fi
 install:
   # install a few of the dependencies that pip would otherwise try to install
   # when intalling scikit-bio
@@ -27,22 +31,37 @@ install:
     'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1'
   - source activate qiita
   - pip install -U pip
-  - pip install sphinx sphinx-bootstrap-theme coveralls 'ipython[all]==2.4.1'
+  - pip install sphinx sphinx-bootstrap-theme 'ipython[all]==2.4.1' nose-timer codecov
   - travis_retry pip install . --process-dependency-links
   - 'echo "backend: Agg" > matplotlibrc'
-script:
+  # Install the biom plugin so we can run the analysis tests
+  - pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
+  - pip install https://github.com/qiita-spots/qtp-biom/archive/master.zip --process-dependency-links
+  - export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
+  - mkdir ~/.qiita_plugins
+  - cp $PWD/qiita_core/support_files/BIOM\ type_2.1.4.conf ~/.qiita_plugins
+before_script:
+    # Some of the tests rely on the plugin system to complete successfuly.
+    # Thus, we need a qiita webserver running to be able to execute the tests.
   - export MOI_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
-  - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then
-    export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
-    export MOI_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
-    fi
+  # EBI, see the end of before_install about why this block is commented out
+  # - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then
+  #   export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
+  #   export MOI_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
+  #   fi
   - ipython profile create qiita-general --parallel
   - qiita-env start_cluster qiita-general
   - qiita-env make --no-load-ontologies
+  - |
+    if [ ${TEST_ADD_STUDIES} == "False" ]; then
+        qiita pet webserver --no-build-docs start &
+    fi
+script:
+  - sleep 5
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi
   - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi
-  - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage -v --cover-package=qiita_db,qiita_pet,qiita_core,qiita_ware; fi
+  - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=$COVER_PACKAGE; fi
   - flake8 qiita_* setup.py scripts/*
   - ls -R /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_pet/support_files/doc/
   - qiita pet webserver
@@ -52,4 +71,4 @@ services:
   - redis-server
   - postgresql
 after_success:
-  - if [ ${TEST_ADD_STUDIES} == "False" ]; then coveralls ; fi
+  - if [ ${TEST_ADD_STUDIES} == "False" ]; then codecov ; fi

README.rst

@@ -80,7 +80,7 @@ future.
 
 .. |Build Status| image:: https://travis-ci.org/biocore/qiita.png?branch=master
    :target: https://travis-ci.org/biocore/qiita
-.. |Coverage Status| image:: https://coveralls.io/repos/biocore/qiita/badge.png?branch=master
-   :target: https://coveralls.io/r/biocore/qiita
+.. |Coverage Status| image:: https://codecov.io/gh/biocore/qiita/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/biocore/qiita
 .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
    :target: https://gitter.im/biocore/qiita?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge

qiita_core/support_files/BIOM type_2.1.4.conf

@@ -0,0 +1,13 @@
+[main]
+NAME = BIOM type
+VERSION = 2.1.4
+DESCRIPTION = The Biological Observation Matrix format
+ENVIRONMENT_SCRIPT = source activate qtp-biom
+START_SCRIPT = start_biom
+PLUGIN_TYPE = artifact definition
+PUBLICATIONS =
+
+[oauth2]
+SERVER_CERT = /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_core/support_files/server.crt
+CLIENT_ID = dHgaXDwq665ksFPqfIoD3Jt8KRXdSioTRa4lGa5mGDnz6JTIBf
+CLIENT_SECRET = xqx61SD4M2EWbaS0WYv3H1nIemkvEAMIn16XMLjy5rTCqi7opCcWbfLINEwtV48bQ

qiita_core/support_files/config_test.cfg

@@ -23,19 +23,19 @@ LOG_DIR =
 REQUIRE_APPROVAL = True
 
 # Base URL: DO NOT ADD TRAILING SLASH
-BASE_URL = https://localhost
+BASE_URL = https://localhost:21174
 
 # Download path files
-UPLOAD_DATA_DIR = /tmp/
+UPLOAD_DATA_DIR = /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_db/support_files/test_data/uploads/
 
 # Working directory path
-WORKING_DIR = /tmp/
+WORKING_DIR = /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_db/support_files/test_data/working_dir/
 
 # Maximum upload size (in Gb)
 MAX_UPLOAD_SIZE = 100
 
 # Path to the base directory where the data files are going to be stored
-BASE_DATA_DIR =
+BASE_DATA_DIR = /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_db/support_files/test_data/
 
 # Valid upload extension, comma separated. Empty for no uploads
 VALID_UPLOAD_EXTENSION = fastq,fastq.gz,txt,tsv,sff,fna,qual

qiita_core/testing.py

@@ -43,9 +43,9 @@ def wait_for_prep_information_job(prep_id, raise_if_none=True):
         else:
             redis_info = loads(r_client.get(job_id))
             while redis_info['status_msg'] == 'Running':
-                sleep(0.05)
+                sleep(0.5)
                 redis_info = loads(r_client.get(job_id))
-        sleep(0.05)
+        sleep(0.5)
 
 
 def wait_for_processing_job(job_id):
@@ -58,5 +58,5 @@ def wait_for_processing_job(job_id):
     """
     job = ProcessingJob(job_id)
     while job.status not in ('success', 'error'):
-        sleep(0.05)
-    sleep(0.5)
+        sleep(1.2)
+    sleep(1.2)

qiita_db/__init__.py

@@ -16,7 +16,6 @@
 import environment_manager
 import exceptions
 import investigation
-import job
 import logger
 import meta_util
 import ontology
@@ -27,11 +26,12 @@
 import study
 import user
 import processing_job
+import private
 
 __version__ = "0.2.0-dev"
 
 __all__ = ["analysis", "artifact", "base", "commands", "environment_manager",
-           "exceptions", "investigation", "job", "logger", "meta_util",
+           "exceptions", "investigation", "logger", "meta_util",
            "ontology", "portal", "reference", "search",
            "software", "sql_connection", "study", "user", "util",
-           "metadata_template", "processing_job"]
+           "metadata_template", "processing_job", "private"]