Fix artifact type creation (#2158)

* Changing the HTML setter to add support dirs

* Extending the patch operation

* Fixing remaining calls to html_summary_fp set

* Fixing test

* Deleting unused function

* Removing unused import

* Fixing tests

* Fixing failing test

* Creating the mountpoint of the new type

* Cleaning up patch and fixing bug

* Fixing tests

* Adding analysis to the list of acceptable parameter types

* Adding the provenance parameter automatically

Author: josenavas

qiita_db/artifact.py

@@ -10,8 +10,8 @@
 from future.utils import viewitems
 from itertools import chain
 from datetime import datetime
-from os import remove
-from os.path import isfile
+from os import remove, makedirs
+from os.path import isfile, exists
 from shutil import rmtree
 from functools import partial
 
@@ -152,6 +152,19 @@ def create_type(name, description, can_be_submitted_to_ebi,
                 [at_id, qdb.util.convert_to_id(fpt, 'filepath_type'), req]
                 for fpt, req in filepath_types]
             qdb.sql_connection.TRN.add(sql, sql_args, many=True)
+
+            # When creating a type is expected that a new mountpoint is created
+            # for that type
+            sql = """INSERT INTO qiita.data_directory
+                        (data_type, mountpoint, subdirectory, active)
+                        VALUES (%s, %s, %s, %s)"""
+            qdb.sql_connection.TRN.add(sql, [name, name, True, True])
+
+            # We are intersted in the dirpath
+            dp = qdb.util.get_mountpoint(name)[0][1]
+            if not exists(dp):
+                makedirs(dp)
+
             qdb.sql_connection.TRN.execute()
 
     @classmethod

qiita_db/software.py

@@ -272,7 +272,7 @@ def create(cls, software, name, description, parameters, outputs=None,
             ptype, dflt = vals
             # Check that the type is one of the supported types
             supported_types = ['string', 'integer', 'float', 'reference',
-                               'boolean', 'prep_template']
+                               'boolean', 'prep_template', 'analysis']
             if ptype not in supported_types and not ptype.startswith(
                     ('choice', 'mchoice', 'artifact')):
                 supported_types.extend(['choice', 'mchoice', 'artifact'])
@@ -337,6 +337,7 @@ def create(cls, software, name, description, parameters, outputs=None,
             sql_type = """INSERT INTO qiita.parameter_artifact_type
                             (command_parameter_id, artifact_type_id)
                           VALUES (%s, %s)"""
+            supported_types = []
             for pname, p_type, atypes in sql_artifact_params:
                 sql_params = [c_id, pname, p_type, True, None]
                 qdb.sql_connection.TRN.add(sql, sql_params)
@@ -345,6 +346,30 @@ def create(cls, software, name, description, parameters, outputs=None,
                     [pid, qdb.util.convert_to_id(at, 'artifact_type')]
                     for at in atypes]
                 qdb.sql_connection.TRN.add(sql_type, sql_params, many=True)
+                supported_types.extend([atid for _, atid in sql_params])
+
+            # If the software type is 'artifact definition', there are a couple
+            # of extra steps
+            if software.type == 'artifact definition':
+                # If supported types is not empty, link the software with these
+                # types
+                if supported_types:
+                    sql = """INSERT INTO qiita.software_artifact_type
+                                    (software_id, artifact_type_id)
+                                VALUES (%s, %s)"""
+                    sql_params = [[software.id, atid]
+                                  for atid in supported_types]
+                    qdb.sql_connection.TRN.add(sql, sql_params, many=True)
+                # If this is the validate command, we need to add the
+                # provenance parameter. This is used internally, that's why
+                # we are adding it here
+                if name == 'Validate':
+                    sql = """INSERT INTO qiita.command_parameter
+                                (command_id, parameter_name, parameter_type,
+                                 required, default_value)
+                             VALUES (%s, 'provenance', 'string', 'False', NULL)
+                             """
+                    qdb.sql_connection.TRN.add(sql, [c_id])
 
             # Add the outputs to the command
             if outputs:

qiita_db/support_files/patches/python_patches/54.py

@@ -392,57 +392,22 @@ def transfer_job(analysis, command_id, params, input_artifact_id, job_data,
 # and (3) taxonomy summary, which will include all the files generated
 # by summarize_taxa_through_plots.py
 
-# Step 1: Create the new type
 with TRN:
-    # Magic number 2 -> The "artifact definition" software type
-    sql = """INSERT INTO qiita.software
-                (name, version, description, environment_script, start_script,
-                software_type_id)
-             VALUES ('Diversity types', '0.1.0',
-                     'Diversity artifacts type plugin',
-                     'source activate qiita', 'start_diversity_types', 2)
-             RETURNING software_id"""
-    TRN.add(sql)
-    divtype_id = TRN.execute_fetchlast()
-
-    # Step 2: Create the validate and HTML generator commands
-    sql = """INSERT INTO qiita.software_command (software_id, name, description)
-             VALUES (%s, %s, %s)
-             RETURNING command_id"""
-    TRN.add(sql, [divtype_id, 'Validate',
-                  'Validates a new artifact of the given diversity type'])
-    validate_cmd_id = TRN.execute_fetchlast()
-    TRN.add(sql, [divtype_id, 'Generate HTML summary',
-                  'Generates the HTML summary of a given diversity type'])
-    html_summary_cmd_id = TRN.execute_fetchlast()
-
-    # Step 3: Add the parameters for the previous commands
-    sql = """INSERT INTO qiita.command_parameter
-                (command_id, parameter_name, parameter_type, required)
-             VALUES (%s, %s, %s, %s)"""
-    sql_args = [(validate_cmd_id, 'files', 'string', True),
-                (validate_cmd_id, 'artifact_type', 'string', True),
-                (validate_cmd_id, 'template', 'prep_template', False),
-                (validate_cmd_id, 'analysis', 'analysis', False),
-                (validate_cmd_id, 'provenance', 'string', False),
-                (html_summary_cmd_id, 'input_data', 'artifact', True)]
-    TRN.add(sql, sql_args, many=True)
-
-    # Step 4: Add the new artifact types
+    # Add the new artifact types
     sql = """INSERT INTO qiita.artifact_type (
                 artifact_type, description, can_be_submitted_to_ebi,
                 can_be_submitted_to_vamps)
              VALUES (%s, %s, %s, %s)
              RETURNING artifact_type_id"""
-    TRN.add(sql, ['distance_matrix', 'Distance matrix holding pairwise '
-                                     'distance between samples', False, False])
+    TRN.add(sql, ['beta_div_plots', 'Qiime 1 beta diversity results',
+                  False, False])
     dm_atype_id = TRN.execute_fetchlast()
     TRN.add(sql, ['rarefaction_curves', 'Rarefaction curves', False, False])
     rc_atype_id = TRN.execute_fetchlast()
     TRN.add(sql, ['taxa_summary', 'Taxa summary plots', False, False])
     ts_atype_id = TRN.execute_fetchlast()
 
-    # Step 5: Associate each artifact with the filetypes that it accepts
+    # Associate each artifact with the filetypes that it accepts
     # At this time we are going to add them as directories, just as it is done
     # right now. We can make it fancier with the new type system.
     # Magic number 8: the filepath_type_id for the directory
@@ -454,36 +419,6 @@ def transfer_job(analysis, command_id, params, input_artifact_id, job_data,
                 [ts_atype_id, 8, True]]
     TRN.add(sql, sql_args, many=True)
 
-    # Step 6: Associate the plugin with the types that it defines
-    sql = """INSERT INTO qiita.software_artifact_type
-                (software_id, artifact_type_id)
-             VALUES (%s, %s)"""
-    sql_args = [[divtype_id, dm_atype_id],
-                [divtype_id, rc_atype_id],
-                [divtype_id, ts_atype_id]]
-    TRN.add(sql, sql_args, many=True)
-
-    # Step 7: Create the new entries for the data directory
-    sql = """INSERT INTO qiita.data_directory
-                (data_type, mountpoint, subdirectory, active)
-             VALUES (%s, %s, %s, %s)"""
-    sql_args = [['distance_matrix', 'distance_matrix', True, True],
-                ['rarefaction_curves', 'rarefaction_curves', True, True],
-                ['taxa_summary', 'taxa_summary', True, True]]
-    TRN.add(sql, sql_args, many=True)
-
-    # Step 8: Give a new client id/client secret pair to the plugins
-    sql = """INSERT INTO qiita.oauth_identifiers (client_id, client_secret)
-                VALUES (%s, %s)"""
-    # Each plugin needs a client id/secret pair, so we are generating it here
-    # at random
-    client_id = get_random_string(50)
-    client_secret = get_random_string(255)
-    TRN.add(sql, [client_id, client_secret])
-    sql = """INSERT INTO qiita.oauth_software (client_id, software_id)
-                VALUES (%s, %s)"""
-    TRN.add(sql, [client_id, divtype_id])
-
     # Create the new commands that execute the current analyses. In qiita,
     # the only commands that where available are Summarize Taxa, Beta
     # Diversity and Alpha Rarefaction. The system was executing rarefaction

qiita_db/test/test_artifact.py

@@ -49,8 +49,7 @@ def test_create_type(self):
                ['Demultiplexed', 'Demultiplexed and QC sequences'],
                ['FASTA', None], ['FASTA_Sanger', None], ['FASTQ', None],
                ['SFF', None], ['per_sample_FASTQ', None],
-               ['distance_matrix', 'Distance matrix holding pairwise '
-                                   'distance between samples'],
+               ['beta_div_plots', 'Qiime 1 beta diversity results'],
                ['rarefaction_curves', 'Rarefaction curves'],
                ['taxa_summary', 'Taxa summary plots']]
         self.assertItemsEqual(obs, exp)
@@ -64,12 +63,12 @@ def test_create_type(self):
                ['Demultiplexed', 'Demultiplexed and QC sequences'],
                ['FASTA', None], ['FASTA_Sanger', None], ['FASTQ', None],
                ['SFF', None], ['per_sample_FASTQ', None],
-               ['distance_matrix', 'Distance matrix holding pairwise '
-                                   'distance between samples'],
+               ['beta_div_plots', 'Qiime 1 beta diversity results'],
                ['rarefaction_curves', 'Rarefaction curves'],
                ['taxa_summary', 'Taxa summary plots'],
                ['NewType', 'NewTypeDesc']]
         self.assertItemsEqual(obs, exp)
+        self.assertTrue(exists(qdb.util.get_mountpoint('NewType')[0][1]))
 
         with self.assertRaises(qdb.exceptions.QiitaDBDuplicateError):
             qdb.artifact.Artifact.create_type(

qiita_db/test/test_util.py

@@ -100,7 +100,7 @@ def test_get_artifact_types(self):
         obs = qdb.util.get_artifact_types()
         exp = {'SFF': 1, 'FASTA_Sanger': 2, 'FASTQ': 3, 'FASTA': 4,
                'per_sample_FASTQ': 5, 'Demultiplexed': 6, 'BIOM': 7,
-               'distance_matrix': 8L, 'rarefaction_curves': 9L,
+               'beta_div_plots': 8L, 'rarefaction_curves': 9L,
                'taxa_summary': 10L}
         self.assertEqual(obs, exp)
 

qiita_pet/handlers/api_proxy/tests/test_artifact.py

@@ -108,8 +108,7 @@ def test_artifact_types_get_req(self):
                          ['FASTQ', None],
                          ['SFF', None],
                          ['per_sample_FASTQ', None],
-                         ['distance_matrix', 'Distance matrix holding pairwise'
-                                             ' distance between samples'],
+                         ['beta_div_plots', 'Qiime 1 beta diversity results'],
                          ['rarefaction_curves', 'Rarefaction curves'],
                          ['taxa_summary', 'Taxa summary plots']]}
 

qiita_pet/handlers/api_proxy/tests/test_processing.py

@@ -55,13 +55,13 @@ def test_list_commands_handler_get_req(self):
         exp = {'status': 'success',
                'message': '',
                'commands': [
-                    {'command': 'Summarize Taxa', 'id': 11,
+                    {'command': 'Summarize Taxa', 'id': 9,
                      'output': [['taxa_summary', 'taxa_summary']]},
-                    {'command': 'Beta Diversity', 'id': 12,
-                     'output': [['distance_matrix', 'distance_matrix']]},
-                    {'command': 'Alpha Rarefaction', 'id': 13,
+                    {'command': 'Beta Diversity', 'id': 10,
+                     'output': [['distance_matrix', 'beta_div_plots']]},
+                    {'command': 'Alpha Rarefaction', 'id': 11,
                      'output': [['rarefaction_curves', 'rarefaction_curves']]},
-                    {'command': 'Single Rarefaction', 'id': 14,
+                    {'command': 'Single Rarefaction', 'id': 12,
                      'output': [['rarefied_table', 'BIOM']]}]}
         # since the order of the commands can change, test them separately
         self.assertItemsEqual(obs.pop('commands'), exp.pop('commands'))