Analysis refactor gui part5

qiita_db/handlers/artifact.py

@@ -80,6 +80,7 @@ def get(self, artifact_id):
         with qdb.sql_connection.TRN:
             artifact = _get_artifact(artifact_id)
             study = artifact.study
+            analysis = artifact.analysis
             response = {
                 'name': artifact.name,
                 'timestamp': str(artifact.timestamp),
@@ -90,7 +91,8 @@ def get(self, artifact_id):
                 'can_be_submitted_to_vamps':
                     artifact.can_be_submitted_to_vamps,
                 'prep_information': [p.id for p in artifact.prep_templates],
-                'study': study.id if study else None}
+                'study': study.id if study else None,
+                'analysis': analysis.id if analysis else None}
             params = artifact.processing_parameters
             response['processing_parameters'] = (
                 params.values if params is not None else None)

qiita_db/handlers/tests/test_artifact.py

@@ -77,10 +77,36 @@ def test_get_artifact(self):
             'is_submitted_to_vamps': None,
             'prep_information': [1],
             'study': 1,
+            'analysis': None,
             'processing_parameters': None,
             'files': exp_fps}
         self.assertEqual(loads(obs.body), exp)
 
+        obs = self.get('/qiita_db/artifacts/9/', headers=self.header)
+        self.assertEqual(obs.code, 200)
+        db_test_raw_dir = qdb.util.get_mountpoint('analysis')[0][1]
+        path_builder = partial(join, db_test_raw_dir)
+        exp_fps = {"biom": [path_builder('1_analysis_18S.biom')]}
+        exp = {
+            'name': 'noname',
+            'visibility': 'sandbox',
+            'type': 'BIOM',
+            'data_type': '18S',
+            'can_be_submitted_to_ebi': False,
+            'ebi_run_accessions': None,
+            'can_be_submitted_to_vamps': False,
+            'is_submitted_to_vamps': None,
+            'prep_information': [],
+            'study': None,
+            'analysis': 1,
+            'processing_parameters': {'biom_table': 8, 'depth': 9000,
+                                      'subsample_multinomial': False},
+            'files': exp_fps}
+        obs = loads(obs.body)
+        # The timestamp is genreated at patch time, so we can't check for it
+        del obs['timestamp']
+        self.assertEqual(obs, exp)
+
     def test_patch(self):
         arguments = {'op': 'add', 'path': '/html_summary/',
                      'value': self.html_fp}

qiita_db/support_files/patches/python_patches/51.py

@@ -7,6 +7,8 @@
 # 3) Transfer all the data in the old structures to the plugin structures
 # 4) Delete old structures
 
+from string import ascii_letters, digits
+from random import SystemRandom
 from os.path import join, exists, basename
 from os import makedirs
 from json import loads
@@ -24,6 +26,24 @@
 # trasnfer the data from the old structure to the new one
 
 
+def get_random_string(length):
+    """Creates a random string of the given length with alphanumeric chars
+
+    Parameters
+    ----------
+    length : int
+        The desired length of the string
+
+    Returns
+    -------
+    str
+        The new random string
+    """
+    sr = SystemRandom()
+    chars = ascii_letters + digits
+    return ''.join(sr.choice(chars) for i in range(length))
+
+
 def create_non_rarefied_biom_artifact(analysis, biom_data, rarefied_table):
     """Creates the initial non-rarefied BIOM artifact of the analysis
 
@@ -95,6 +115,7 @@ def create_non_rarefied_biom_artifact(analysis, biom_data, rarefied_table):
         TRN.add(sql, [analysis['timestamp'], biom_data['data_type_id'],
                       4, 7, False])
         artifact_id = TRN.execute_fetchlast()
+
         # Associate the artifact with the analysis
         sql = """INSERT INTO qiita.analysis_artifact
                     (analysis_id, artifact_id)
@@ -398,6 +419,9 @@ def transfer_job(analysis, command_id, params, input_artifact_id, job_data,
              VALUES (%s, %s, %s, %s)"""
     sql_args = [(validate_cmd_id, 'files', 'string', True),
                 (validate_cmd_id, 'artifact_type', 'string', True),
+                (validate_cmd_id, 'template', 'prep_template', False),
+                (validate_cmd_id, 'analysis', 'analysis', False),
+                (validate_cmd_id, 'provenance', 'string', False),
                 (html_summary_cmd_id, 'input_data', 'artifact', True)]
     TRN.add(sql, sql_args, many=True)
 
@@ -445,6 +469,18 @@ def transfer_job(analysis, command_id, params, input_artifact_id, job_data,
                 ['taxa_summary', 'taxa_summary', True, True]]
     TRN.add(sql, sql_args, many=True)
 
+    # Step 8: Give a new client id/client secret pair to the plugins
+    sql = """INSERT INTO qiita.oauth_identifiers (client_id, client_secret)
+                VALUES (%s, %s)"""
+    # Each plugin needs a client id/secret pair, so we are generating it here
+    # at random
+    client_id = get_random_string(50)
+    client_secret = get_random_string(255)
+    TRN.add(sql, [client_id, client_secret])
+    sql = """INSERT INTO qiita.oauth_software (client_id, software_id)
+                VALUES (%s, %s)"""
+    TRN.add(sql, [client_id, divtype_id])
+
     # Create the new commands that execute the current analyses. In qiita,
     # the only commands that where available are Summarize Taxa, Beta
     # Diversity and Alpha Rarefaction. The system was executing rarefaction
@@ -491,8 +527,8 @@ def transfer_job(analysis, command_id, params, input_artifact_id, job_data,
         (sum_taxa_cmd_id, 'sort', 'bool', False, 'False'),
         # Beta Diversity
         (bdiv_cmd_id, 'tree', 'string', False, ''),
-        (bdiv_cmd_id, 'metrics',
-         'mchoice:["abund_jaccard","binary_chisq","binary_chord",'
+        (bdiv_cmd_id, 'metric',
+         'choice:["abund_jaccard","binary_chisq","binary_chord",'
          '"binary_euclidean","binary_hamming","binary_jaccard",'
          '"binary_lennon","binary_ochiai","binary_otu_gain","binary_pearson",'
          '"binary_sorensen_dice","bray_curtis","bray_curtis_faith",'
@@ -501,12 +537,21 @@ def transfer_job(analysis, command_id, params, input_artifact_id, job_data,
          '"pearson","soergel","spearman_approx","specprof","unifrac",'
          '"unifrac_g","unifrac_g_full_tree","unweighted_unifrac",'
          '"unweighted_unifrac_full_tree","weighted_normalized_unifrac",'
-         '"weighted_unifrac"]', False, '["binary_jaccard","bray_curtis"]'),
+         '"weighted_unifrac"]', False, '"binary_jaccard"'),
         # Alpha rarefaction
         (arare_cmd_id, 'tree', 'string', False, ''),
         (arare_cmd_id, 'num_steps', 'integer', False, 10),
         (arare_cmd_id, 'min_rare_depth', 'integer', False, 10),
         (arare_cmd_id, 'max_rare_depth', 'integer', False, 'Default'),
+        (arare_cmd_id, 'metrics',
+         'mchoice:["ace","berger_parker_d","brillouin_d","chao1","chao1_ci",'
+         '"dominance","doubles","enspie","equitability","esty_ci",'
+         '"fisher_alpha","gini_index","goods_coverage","heip_e",'
+         '"kempton_taylor_q","margalef","mcintosh_d","mcintosh_e",'
+         '"menhinick","michaelis_menten_fit","observed_otus",'
+         '"observed_species","osd","simpson_reciprocal","robbins",'
+         '"shannon","simpson","simpson_e","singles","strong","PD_whole_tree"]',
+         False, '["chao1","observed_otus"]'),
         # Single rarefaction
         (srare_cmd_id, 'depth', 'integer', True, None),
         (srare_cmd_id, 'subsample_multinomial', 'bool', False, 'False')
@@ -553,6 +598,22 @@ def transfer_job(analysis, command_id, params, input_artifact_id, job_data,
     TRN.add(sql, ['rarefied_table', srare_cmd_id, biom_atype_id])
     srare_cmd_out_id = TRN.execute_fetchlast()
 
+    # Step 6: Add default parameter sets
+    sql = """INSERT INTO qiita.default_parameter_set
+                (command_id, parameter_set_name, parameter_set)
+             VALUES (%s, %s, %s)"""
+    sql_args = [
+        [sum_taxa_cmd_id, 'Defaults',
+         '{"sort": false, "metadata_category": ""}'],
+        [bdiv_cmd_id, 'Unweighted UniFrac',
+         '{"metrics": "unweighted_unifrac", "tree": ""}'],
+        [arare_cmd_id, 'Defaults',
+         '{"max_rare_depth": "Default", "tree": "", "num_steps": 10, '
+         '"min_rare_depth": 10, "metrics": ["chao1", "observed_otus"]}'],
+        [srare_cmd_id, 'Defaults',
+         '{"subsample_multinomial": "False"}']]
+    TRN.add(sql, sql_args, many=True)
+
 # At this point we are ready to start transferring the data from the old
 # structures to the new structures. Overview of the procedure:
 # Step 1: Add initial set of artifacts up to rarefied table

qiita_pet/handlers/artifact_handlers/__init__.py

@@ -7,5 +7,6 @@
 # -----------------------------------------------------------------------------
 
 from .base_handlers import ArtifactSummaryAJAX, ArtifactAJAX
+from .process_handlers import ProcessArtifactHandler
 
-__all__ = ['ArtifactSummaryAJAX', 'ArtifactAJAX']
+__all__ = ['ArtifactSummaryAJAX', 'ArtifactAJAX', 'ProcessArtifactHandler']

qiita_pet/templates/analysis_description.html

@@ -25,6 +25,28 @@
       location.reload();
   }
 
+  function reload_UI_post_workflow_submission(artifact_id) {
+      $("#processing-content-div").hide();
+      $("#processing-content-div").html("");
+      $("#analysis-content-div").show();
+      populateContentArtifact(artifact_id);
+  }
+
+  function load_process_artifact_ui(artifact_id) {
+    $("#analysis-content-div").hide()
+    $("#processing-content-div").show()
+    show_loading('{% raw qiita_config.portal_dir %}', 'processing-content-div');
+    $.get('{% raw qiita_config.portal_dir %}/artifact/' + artifact_id + '/process/', function(data){
+      $("#processing-content-div").html(data);
+    })
+      .fail(function(object, status, error_msg) {
+        bootstrapAlert("Error loading processing page: " + status + " " + object.statusText, "danger");
+        $("#analysis-content-div").show()
+        $("#processing-content-div").hide()
+      }
+    )
+  };
+
   /**
    * Creates the HTML with the job information and adds it to `target`
    *
@@ -193,6 +215,7 @@
 
     // Add the vue object to the div, so we avoid to have global variables
     $("#analysis-network-div").data('data-graph-vue', vueGraph);
+    $("processing-content-div").hide();
     {% if alert_msg %}
       bootstrapAlert("{{alert_msg}}", "{{alert_type}}");
     {% end %}
@@ -214,22 +237,25 @@ <h2>{{analysis_name}} - ID {{analysis_id}}</h2>
     <h3>{{analysis_description}}</h3>
   </div>
 </div>
-<div class="row">
-  <div class="col">
-    <h4><a class="btn btn-info" id="show-hide-network-btn" onclick="toggle_network_graph();">-</a><i> Processing network</i></h4>
-    <b>(Click nodes for more information, blue are jobs)</b>
+<div class="row" id='analysis-content-div'>
+  <div class="row">
+    <div class="col">
+      <h4><a class="btn btn-info" id="show-hide-network-btn" onclick="toggle_network_graph();">-</a><i> Processing network</i></h4>
+      <b>(Click nodes for more information, blue are jobs)</b>
+    </div>
   </div>
-</div>
-<div class='row' id="analysis-graph-vue">
-  <analysis-graph></analysis-graph>
-</div>
-<div class='row'>
-  <div class='col-md-12' style='width:90%' id='analysis-job-div'>
+  <div class='row' id="analysis-graph-vue">
+    <analysis-graph></analysis-graph>
   </div>
-</div>
-<div class='row'>
-  <div class='col-md-12' style='width:90%' id='analysis-results'>
+  <div class='row'>
+    <div class='col-md-12' style='width:90%' id='analysis-job-div'>
+    </div>
+  </div>
+  <div class='row'>
+    <div class='col-md-12' style='width:90%' id='analysis-results'>
+    </div>
   </div>
 </div>
+<div class="row" id='processing-content-div'></div>
 
 {% end %}

qiita_pet/templates/artifact_ajax/artifact_summary.html

@@ -96,7 +96,7 @@ <h4>
       <i id='summary-title'>{{name}}</i><i> (ID: {{artifact_id}}) Visibility: {{visibility}}</i>
       {% if editable %}
         <a class="btn btn-default btn-sm" data-toggle="modal" data-target="#update-artifact-name"><span class="glyphicon glyphicon-pencil"></span> Edit</a>
-        <a class="btn btn-default btn-sm" onclick="populate_main_div('{% raw qiita_config.portal_dir %}/study/process/', {artifact_id: {{artifact_id}} });"><span class="glyphicon glyphicon-play"></span> Process</a>
+        <a class="btn btn-default btn-sm" onclick="load_process_artifact_ui({{artifact_id}});"><span class="glyphicon glyphicon-play"></span> Process</a>
         <a class="btn btn-danger btn-sm" onclick="if (confirm('Are you sure you want to delete artifact {{artifact_id}}?')){ delete_artifact({{artifact_id}}) };"><span class="glyphicon glyphicon-trash"></span> Delete</a>
         {% raw buttons %}
       {% end %}

qiita_pet/templates/study_ajax/prep_summary.html

@@ -26,6 +26,10 @@
       populate_main_div('{% raw qiita_config.portal_dir %}/study/description/prep_template/', { prep_id: {{prep_id}}, study_id: {{study_id}} });
   };
 
+  function load_process_artifact_ui(artifact_id) {
+      populate_main_div('{% raw qiita_config.portal_dir %}/study/process/', {artifact_id: artifact_id });
+  };
+
   /*
    * Function to draw a vis.Network
    *

qiita_pet/templates/study_base.html

@@ -6,6 +6,11 @@
 <link rel="stylesheet" src="{% raw qiita_config.portal_dir %}/static/vendor/css/vis-network.min.css" type="text/css">
 <script type="text/javascript" src="{% raw qiita_config.portal_dir %}/static/vendor/js/vis.min.js"></script>
 <script type="text/javascript">
+
+  function reload_UI_post_workflow_submission(artifact_id) {
+    populate_main_div('{% raw qiita_config.portal_dir %}/study/description/baseinfo/', { study_id: {{study_info['study_id']}} });
+  }
+
   var network = null;
   // Track what prep template is currently being shown.
   var curr_prep = -1;

qiita_pet/webserver.py

@@ -27,14 +27,14 @@
     StudyEditHandler, ListStudiesHandler, SearchStudiesAJAX, EBISubmitHandler,
     CreateStudyAJAX, ShareStudyAJAX, StudyApprovalList, ArtifactGraphAJAX,
     VAMPSHandler, PrepTemplateGraphAJAX,
-    ProcessArtifactHandler, ListCommandsHandler, ListOptionsHandler,
+    ListCommandsHandler, ListOptionsHandler,
     PrepTemplateAJAX, NewArtifactHandler, SampleAJAX,
     StudyDeleteAjax, ArtifactAdminAJAX,
     NewPrepTemplateAjax, DataTypesMenuAJAX, StudyFilesAJAX,
     PrepTemplateSummaryAJAX,
     WorkflowHandler, WorkflowRunHandler, JobAJAX, AutocompleteHandler)
 from qiita_pet.handlers.artifact_handlers import (
-    ArtifactSummaryAJAX, ArtifactAJAX)
+    ArtifactSummaryAJAX, ArtifactAJAX, ProcessArtifactHandler)
 from qiita_pet.handlers.websocket_handlers import (
     MessageHandler, SelectedSocketHandler, SelectSamplesHandler)
 from qiita_pet.handlers.logger_handlers import LogEntryViewerHandler
@@ -118,7 +118,6 @@ def __init__(self):
             (r"/study/process/workflow/run/", WorkflowRunHandler),
             (r"/study/process/workflow/", WorkflowHandler),
             (r"/study/process/job/", JobAJAX),
-            (r"/study/process/", ProcessArtifactHandler),
             (r"/study/list/socket/", SelectSamplesHandler),
             (r"/study/search/(.*)", SearchStudiesAJAX),
             (r"/study/new_artifact/", NewArtifactHandler),
@@ -130,6 +129,7 @@ def __init__(self):
             # Artifact handlers
             (r"/artifact/graph/", ArtifactGraphAJAX),
             (r"/artifact/(.*)/summary/", ArtifactSummaryAJAX),
+            (r"/artifact/(.*)/process/", ProcessArtifactHandler),
             (r"/artifact/(.*)/", ArtifactAJAX),
             (r"/prep_template/", PrepTemplateHandler),
             (r"/ontology/", OntologyHandler),