diff --git a/.travis.yml b/.travis.yml index 5a191929f..d1a1bad90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: - TEST_ADD_STUDIES=True before_install: - redis-server --version + - redis-server /etc/redis/redis.conf --port 7777 & - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - ./miniconda.sh -b @@ -28,9 +29,14 @@ install: - source activate qiita - pip install -U pip - pip install sphinx sphinx-bootstrap-theme coveralls 'ipython[all]==2.4.1' - - travis_retry pip install . --process-dependency-links - 'echo "backend: Agg" > matplotlibrc' -script: + - git clone https://github.com/nicolasff/webdis + - pushd webdis + - make + - ./webdis & + - popd + - travis_retry pip install . --process-dependency-links +before_script: - export MOI_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg; @@ -39,6 +45,15 @@ script: - ipython profile create qiita-general --parallel - qiita-env start_cluster qiita-general - qiita-env make --no-load-ontologies + # loading redbiom with Qiita's test set + # first let's make sure redis is empty + - curl -s http://127.0.0.1:7379/FLUSHALL > /dev/null + - redbiom admin create-context --name "qiita-test" --description "qiita-test context" + - redbiom admin load-sample-metadata --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt + - redbiom admin load-sample-metadata-search --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt + - redbiom admin load-observations --table `pwd`/qiita_db/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test + - redbiom admin load-sample-data --table `pwd`/qiita_db/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test +script: - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi diff --git a/INSTALL.md b/INSTALL.md index d09f69f12..0fd1c680a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -60,6 +60,7 @@ Install the non-python dependencies * [PostgreSQL](http://www.postgresql.org/download/) (minimum required version 9.3.5, we have tested most extensively with 9.3.6) * [redis-server](http://redis.io) (we have tested most extensively with 2.8.17) +* [webdis] (https://github.com/nicolasff/webdis) (latest version should be fine but we have tested the most with 9ee6fe2 - Feb 6, 2016) There are several options to install these dependencies depending on your needs: @@ -87,6 +88,28 @@ brew update brew install homebrew/versions/redis28 ``` +### webdis + +Note that this is the only package that assumes that Qiita is already installed (due to library dependencies). Also, that the general suggestion is to have 2 redis servers running, one for webdis/redbiom and the other for Qiita. The reason for multiple redis servers is so that the redbiom cache can be flushed without impacting the operation of the qiita server itself. + +The following instructions install, compile and pre-populates the redbiom redis DB so we assume that redis is running on the default port and that Qiita is fully installed as the redbiom package is installed with Qiita. + +``` +git clone https://github.com/nicolasff/webdis +pushd webdis +make +./webdis & +popd +# note that this assumes that Qiita is already installed +fp=`python -c 'import qiita_db; print qiita_db.__file__'` +qdbd=`dirname $fp` +redbiom admin create-context --name "qiita-test" --description "qiita-test context" +redbiom admin load-sample-metadata --metadata ${qdbd}/support_files/test_data/templates/1_19700101-000000.txt +redbiom admin load-sample-metadata-search --metadata ${qdbd}/support_files/test_data/templates/1_19700101-000000.txt +redbiom admin load-observations --table ${qdbd}/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test +redbiom admin load-sample-data --table ${qdbd}/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test +``` + Install Qiita development version and its python dependencies ------------------------------------------------------------- @@ -163,6 +186,12 @@ Next, make a test environment: qiita-env make --no-load-ontologies ``` +Finally, redbiom relies on the REDBIOM_HOST environment variable to set the URL to query. By default is set to http://127.0.0.1:7379, which is the webdis default. For example you could: + +```bash +export REDBIOM_HOST=http://my_host.com:7329 +``` + ## Start Qiita Start postgres (instructions vary depending on operating system and install method). diff --git a/qiita_pet/handlers/qiita_redbiom.py b/qiita_pet/handlers/qiita_redbiom.py new file mode 100644 index 000000000..74e18e1cc --- /dev/null +++ b/qiita_pet/handlers/qiita_redbiom.py @@ -0,0 +1,173 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from requests import ConnectionError +import redbiom.summarize +import redbiom.search +import redbiom._requests +import redbiom.util +import redbiom.fetch +from tornado.gen import coroutine, Task + +from qiita_core.util import execute_as_transaction + +from .base_handlers import BaseHandler + + +class RedbiomPublicSearch(BaseHandler): + @execute_as_transaction + def get(self, search): + self.render('redbiom.html') + + @execute_as_transaction + def _redbiom_search(self, query, search_on, callback): + error = False + message = '' + results = [] + + try: + df = redbiom.summarize.contexts() + except ConnectionError: + error = True + message = 'Redbiom is down - contact admin, thanks!' + + if not error: + contexts = df.ContextName.values + query = query.lower() + features = [] + + if search_on in ('metadata', 'categories'): + try: + features = redbiom.search.metadata_full( + query, categories=(search_on == 'categories')) + except TypeError: + error = True + message = ( + 'Not a valid search: "%s", are you sure this is a ' + 'valid metadata %s?' % ( + query, 'value' if search_on == 'metadata' else + 'category')) + except ValueError: + error = True + message = ( + 'Not a valid search: "%s", your query is too small ' + '(too few letters), try a longer query' % query) + elif search_on == 'observations': + features = [s.split('_', 1)[1] for context in contexts + for s in redbiom.util.samples_from_observations( + query.split(' '), True, context)] + else: + error = True + message = ('Incorrect search by: you can use observations ' + 'or metadata and you passed: %s' % search_on) + + if not error: + import qiita_db as qdb + import qiita_db.sql_connection as qdbsc + if features: + if search_on in ('metadata', 'observations'): + sql = """ + WITH main_query AS ( + SELECT study_title, study_id, artifact_id, + array_agg(DISTINCT sample_id) AS samples, + qiita.artifact_descendants(artifact_id) AS + children + FROM qiita.study_prep_template + JOIN qiita.prep_template USING (prep_template_id) + JOIN qiita.prep_template_sample USING + (prep_template_id) + JOIN qiita.study USING (study_id) + WHERE sample_id IN %s + GROUP BY study_title, study_id, artifact_id) + SELECT study_title, study_id, samples, + name, command_id, + (main_query.children).artifact_id AS artifact_id + FROM main_query + JOIN qiita.artifact a ON + (main_query.children).artifact_id = a.artifact_id + JOIN qiita.artifact_type at ON ( + at.artifact_type_id = a.artifact_type_id + AND artifact_type = 'BIOM') + ORDER BY artifact_id + """ + with qdbsc.TRN: + qdbsc.TRN.add(sql, [tuple(features)]) + results = [] + commands = {} + for row in qdbsc.TRN.execute_fetchindex(): + title, sid, samples, name, cid, aid = row + nr = {'study_title': title, 'study_id': sid, + 'artifact_id': aid, 'aname': name, + 'samples': samples} + if cid is not None: + if cid not in commands: + c = qdb.software.Command(cid) + commands[cid] = { + 'sfwn': c.software.name, + 'sfv': c.software.version, + 'cmdn': c.name + } + nr['command'] = commands[cid]['cmdn'] + nr['software'] = commands[cid]['sfwn'] + nr['version'] = commands[cid]['sfv'] + else: + nr['command'] = None + nr['software'] = None + nr['version'] = None + results.append(nr) + else: + sql = """ + WITH get_studies AS ( + SELECT + trim(table_name, 'sample_')::int AS + study_id, + array_agg(column_name::text) AS columns + FROM information_schema.columns + WHERE column_name IN %s + AND table_name LIKE 'sample_%%' + AND table_name NOT IN ( + 'prep_template', + 'prep_template_sample') + GROUP BY table_name) + SELECT study_title, get_studies.study_id, columns + FROM get_studies + JOIN qiita.study ON get_studies.study_id = + qiita.study.study_id""" + with qdbsc.TRN: + results = [] + qdbsc.TRN.add(sql, [tuple(features)]) + for row in qdbsc.TRN.execute_fetchindex(): + title, sid, cols = row + nr = {'study_title': title, 'study_id': sid, + 'artifact_id': None, 'aname': None, + 'samples': cols, + 'command': ', '.join(cols), + 'software': None, 'version': None} + results.append(nr) + else: + error = True + message = 'No samples where found! Try again ...' + callback((results, message)) + + @coroutine + @execute_as_transaction + def post(self, search): + search = self.get_argument('search', None) + search_on = self.get_argument('search_on', None) + + data = [] + if search is not None and search and search != ' ': + if search_on in ('observations', 'metadata', 'categories'): + data, msg = yield Task( + self._redbiom_search, search, search_on) + else: + msg = 'Not a valid option for search_on' + else: + msg = 'Nothing to search for ...' + + self.write({'status': 'success', 'message': msg, 'data': data}) diff --git a/qiita_pet/static/js/qiita.js b/qiita_pet/static/js/qiita.js index 8b8aa56e0..a2a20c6e7 100644 --- a/qiita_pet/static/js/qiita.js +++ b/qiita_pet/static/js/qiita.js @@ -45,16 +45,22 @@ function bootstrapAlert(message, severity, timeout){ } } +/* + * format_extra_info_processing_jobs will add new rows to the study lists + * + * @param message: data, the original data object for the row + * 0: blank +/- button + * 1: heartbeat + * 2: name + * 3: status + * 4: step + * 5: id + * 6: params + * 7: processing_job_workflow_id + * + */ + function format_extra_info_processing_jobs ( data ) { - // `data` is the original data object for the row - // 0: blank +/- button - // 1: heartbeat - // 2: name - // 3: status - // 4: step - // 5: id - // 6: params - // 7: processing_job_workflow_id let row = ''+ ''+ @@ -77,11 +83,25 @@ function format_extra_info_processing_jobs ( data ) { return row } +/* + * show_hide toggles visibility for the given div + * + * @param message: div, the div to toggle visibility + * + */ function show_hide(div) { $('#' + div).toggle(); } +/* + * delete_analysis will delete an analysis + * + * @param name: The name of the analysis to delete + * @param analysis_id: The id of the analysis to delete + * + */ + function delete_analysis(aname, analysis_id) { if (confirm('Are you sure you want to delete analysis: ' + aname + '?')) { var form = $("") @@ -100,6 +120,10 @@ function delete_analysis(aname, analysis_id) { } } +/* + * show_hide_process_list will toggle the process/job listing visibility + */ + function show_hide_process_list() { if ($("#qiita-main").width() == $("#qiita-main").parent().width()) { // let's update the job list @@ -117,3 +141,177 @@ function show_hide_process_list() { $("#qiita-processing").hide(); } } + +/* + * send_samples_to_moi send the selected samples for the given artifact ids to moi + * + * @param aids: A list of artifact ids to add + * @param samples: The ids of the samples to add + * + * Note that we have a list of artifact ids cause the user can select one single + * artifact to add or all study artifacts + */ + +function send_samples_to_moi(aids, samples) { + bootstrapAlert('We are adding ' + aids.length + ' artifact(s) to the analysis. This ' + + 'might take some time based on the number of samples on each artifact.', "warning", 10000); + + if (typeof samples === 'undefined') { + $.get('/artifact/samples/', {ids:aids}) + .done(function ( data ) { + if (data['status']=='success') { + moi.send('sel', data['data']); + } else { + bootstrapAlert('ERROR: ' + data['msg'], "danger", 10000); + } + }); + } else { + var data = {} + data[aids[0]] = samples + moi.send('sel', data); + } +} + +/* + * redbiom_send_to_moi adds an artifact to moi + * + * @param aid: the artifact id to add + * @param row: the DataTable row from redbiom so we can get the samples ids + * + */ + +function redbiom_send_to_moi(aid, row) { + var row_data = $('#redbiom-table').dataTable().fnGetData(row); + send_samples_to_moi([aid], row_data.samples); +} + +/* + * sel_study selects all artifact ids from a study list DataTable row and + * sends it to moi + * + * @param name: the DataTable div name + * @param row: the DataTable row to select artifact ids + * + */ + +function sel_study(name, row) { + var row_data = $('#'+name).dataTable().fnGetData(row); + var aids = [] + + for(var i=0;iServer communication error. Sample removal will not be recorded. Please try again later."); +}; + +/* + * display_number_of_samples_added is a callback moi funtion that shows how many samples were added + * + * @param num_samples: the number of samples + * + */ + +function display_number_of_samples_added(num_samples) { + bootstrapAlert(num_samples + ' samples selected.', "success", 10000); + $('#dflt-sel-info').css('color', 'rgb(0, 160, 0)'); +} + +/** + * + * Function to show the loading gif in a given div + * + * @param div_name string with the name of the div to populate with the loading gif + * + * This function replaces the content of the given div with the + * gif to show that the section of page is loading + * + */ +function show_loading(div_name) { + $("#" + div_name).html(""); +} diff --git a/qiita_pet/templates/analysis_results.html b/qiita_pet/templates/analysis_results.html index b8e1674fd..74b40443f 100644 --- a/qiita_pet/templates/analysis_results.html +++ b/qiita_pet/templates/analysis_results.html @@ -3,8 +3,6 @@ {% extends sitebase.html%} {%block head%} - - - - + + +{%end%} + +{%block content%} + Some general instructions! +

+ +
+
+ +
+ +
+ +
+
+ +
+
+ + +
+ +
+
+
+
+
+ + + + + + + + + +
+ + +{% end %} diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 80d037d7e..9477b5379 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -34,6 +34,7 @@ + {% if portal_styling.custom_css %}