Skip to content

Improve workflows #3083

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Mar 17, 2021
Merged

Improve workflows #3083

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
3b934a1
fix #3076
antgonza Feb 19, 2021
fb17bd5
addressing @ElDeveloper comments
antgonza Feb 23, 2021
be9848a
Merge branch 'dev' of github.com:biocore/qiita into workflows-GUI
antgonza Feb 23, 2021
51cc864
adding workflows GUI
antgonza Feb 23, 2021
d7655ae
improve HTML
antgonza Feb 23, 2021
aa82154
db changes and improvements
antgonza Feb 25, 2021
189ffd0
Merge branch 'dev' of github.com:biocore/qiita into workflows-GUI
antgonza Mar 3, 2021
35ed05e
comment out new db changes
antgonza Mar 3, 2021
e0a05c0
rm default_workflows from PluginHandler.get
antgonza Mar 3, 2021
e6be223
improving workflows after deployment
antgonza Mar 9, 2021
d4c5153
fix error
antgonza Mar 9, 2021
fbcaa6f
merging dev
antgonza Mar 12, 2021
400e07a
clean up
antgonza Mar 12, 2021
d8f2cf4
Merge branch 'dev' of github.com:biocore/qiita into improve-workflows
antgonza Mar 13, 2021
090610d
minor fixes
antgonza Mar 13, 2021
1426f7d
improving html
antgonza Mar 13, 2021
3df3dc6
comments from @justinshaffer
antgonza Mar 16, 2021
b8adbbb
addressing @ElDeveloper comments
antgonza Mar 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion qiita_db/processing_job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1974,7 +1974,7 @@ def from_default_workflow(cls, user, dflt_wf, req_params, name=None,
parent_ids.append(source_id)
# Get the connections between the job and the source
connections = data['connections'].connections
for out, in_param in connections:
for out, in_param, _ in connections:
# We take advantage of the fact the parameters are
# stored in JSON to encode the name of the output
# artifact from the previous job
Expand Down
44 changes: 38 additions & 6 deletions qiita_db/software.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1783,12 +1783,13 @@ def connections(self):
the destination command.
"""
with qdb.sql_connection.TRN:
sql = """SELECT name, parameter_name
sql = """SELECT name, parameter_name, artifact_type
FROM qiita.default_workflow_edge_connections c
JOIN qiita.command_output o
ON c.parent_output_id = o.command_output_id
JOIN qiita.command_parameter p
ON c.child_input_id = p.command_parameter_id
LEFT JOIN qiita.artifact_type USING (artifact_type_id)
WHERE default_workflow_edge_id = %s"""
qdb.sql_connection.TRN.add(sql, [self.id])
return qdb.sql_connection.TRN.execute_fetchindex()
Expand Down Expand Up @@ -1832,8 +1833,8 @@ def iter(cls, active=True):
def active(self):
"""Retrieves active status of the default workflow

Retruns
----------
Returns
-------
active : bool
active value
"""
Expand Down Expand Up @@ -1866,9 +1867,38 @@ def name(self):
qdb.sql_connection.TRN.add(sql, [self.id])
return qdb.sql_connection.TRN.execute_fetchlast()

@property
def description(self):
"""Retrieves the description of the default workflow

Returns
-------
str
description value
"""
with qdb.sql_connection.TRN:
sql = """SELECT description
FROM qiita.default_workflow
WHERE default_workflow_id = %s"""
qdb.sql_connection.TRN.add(sql, [self.id])
return qdb.sql_connection.TRN.execute_fetchlast()

@description.setter
def description(self, description):
"""Changes the description of the default workflow

Parameters
----------
description : str
New description value
"""
sql = """UPDATE qiita.default_workflow SET description = %s
WHERE default_workflow_id = %s"""
qdb.sql_connection.perform_as_transaction(sql, [description, self._id])

@property
def data_type(self):
"""Retrieves all the data_types the workflow accepts
"""Retrieves all the data_types accepted by the default workflow

Returns
----------
Expand Down Expand Up @@ -1897,7 +1927,8 @@ def graph(self):
# Retrieve all graph workflow nodes
sql = """SELECT default_workflow_node_id
FROM qiita.default_workflow_node
WHERE default_workflow_id = %s"""
WHERE default_workflow_id = %s
ORDER BY default_workflow_node_id"""
qdb.sql_connection.TRN.add(sql, [self.id])
db_nodes = qdb.sql_connection.TRN.execute_fetchflatten()

Expand All @@ -1910,7 +1941,8 @@ def graph(self):
JOIN qiita.default_workflow_node n
ON e.parent_id = n.default_workflow_node_id
OR e.child_id = n.default_workflow_node_id
WHERE default_workflow_id = %s"""
WHERE default_workflow_id = %s
ORDER BY default_workflow_edge_id"""
qdb.sql_connection.TRN.add(sql, [self.id])
db_edges = qdb.sql_connection.TRN.execute_fetchindex()

Expand Down
12 changes: 11 additions & 1 deletion qiita_db/support_files/patches/81.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ ALTER TABLE qiita.prep_template ADD modification_timestamp TIMESTAMP DEFAULT CUR

-- a. Removing software_id from qiita.default_workflow and replacing it by a
-- table which will like different data_types with the default_workflow +
-- adding an active flag in case we need to deprecate default_workflows
-- adding an active flag in case we need to deprecate default_workflows +
-- adding a description column
ALTER TABLE qiita.default_workflow DROP software_id;
CREATE TABLE qiita.default_workflow_data_type (
default_workflow_id BIGINT NOT NULL,
Expand All @@ -20,6 +21,7 @@ CREATE TABLE qiita.default_workflow_data_type (
PRIMARY KEY(default_workflow_id, data_type_id)
);
ALTER TABLE qiita.default_workflow ADD active BOOL DEFAULT TRUE;
ALTER TABLE qiita.default_workflow ADD description TEXT;

-- b. Removing command_id from qiita.default_workflow_node and default_parameter_set as this information
-- can be accessed via the default_parameter object (the info is duplicated)
Expand All @@ -37,3 +39,11 @@ INSERT INTO qiita.default_workflow_data_type (default_workflow_id, data_type_id)
(1, 2),
(2, 2),
(3, 3);

-- d. adding descriptions
UPDATE qiita.default_workflow
SET description = 'This accepts html <a href="https://qiita.ucsd.edu">Qiita!</a><br/><br/><b>BYE!</b>'
WHERE default_workflow_id = 1;
UPDATE qiita.default_workflow
SET description = 'This is another description'
WHERE default_workflow_id = 2;
3 changes: 2 additions & 1 deletion qiita_db/support_files/qiita-db.dbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,7 @@
<column name="default_workflow_id" type="bigserial" jt="-5" mandatory="y" />
<column name="name" type="varchar" jt="12" mandatory="y" />
<column name="active" type="boolean" jt="-7" />
<column name="description" type="text" jt="12" />
<index name="pk_default_workflow" unique="PRIMARY_KEY" >
<column name="default_workflow_id" />
</index>
Expand Down Expand Up @@ -1710,7 +1711,7 @@ Controlled Vocabulary]]></comment>
<entity schema="qiita" name="data_directory" color="B2CDF7" x="576" y="928" />
<entity schema="qiita" name="data_type" color="D0DEF5" x="704" y="1120" />
<entity schema="qiita" name="default_parameter_set" color="B2CDF7" x="2416" y="1104" />
<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2672" y="1136" />
<entity schema="qiita" name="default_workflow" color="B2CDF7" x="2656" y="1120" />
<entity schema="qiita" name="default_workflow_data_type" color="C1D8EE" x="2640" y="1008" />
<entity schema="qiita" name="default_workflow_edge" color="B2CDF7" x="2656" y="1248" />
<entity schema="qiita" name="default_workflow_edge_connections" color="B2CDF7" x="2576" y="1408" />
Expand Down
45 changes: 26 additions & 19 deletions qiita_db/support_files/qiita-db.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<!DOCTYPE html>
<html lang='en-us' xmlns='http://www.w3.org/1999/xhtml' >
<head>
<title>qiita&#45;db</title>
<title>qiita</title>
<meta http-equiv='Content-Type' content='text/html; charset=UTF-8'/>
<meta name='viewport' content='width=device-width, initial-scale=1, shrink-to-fit=no' />
<link rel='shortcut icon' href='https://dbschema.com/images/favicon.ico'>
Expand Down Expand Up @@ -299,7 +299,7 @@
<!-- == Legend == -->
<g transform='translate(10,10)'>
<rect x='10' y='10' width='230' height='56' rx='6' ry='6' style='fill:url(#legendGradient);filter:url(#shadow);'/>
<text x='20' y='25' class='legendTitle' >qiita-db</text>
<text x='20' y='25' class='legendTitle' >qiita</text>
<a xlink:href='https://dbschema.com'><text x='20' y='39' class='legendSubTitle'><tspan>(c) DbSchema</tspan></text></a>
<text x='20' y='53' class='legendSubTitle'><tspan>Hover columns to read the comments.</tspan></text>
</g>
Expand Down Expand Up @@ -629,13 +629,13 @@
default_parameter_set ref software_command ( command_id )</title></path>
<text x='2445' y='1218' transform='rotate(90 2445,1218)' class='relName'>command_id</text>
<!-- == Fk 'default_workflow_data_type_fk_default_workflow_id' == -->
<path id='default_workflow_data_type_fk_default_workflow_id' onmouseover="hghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" onmouseout="uhghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" transform='translate(8,0)' class='scene' d='M 2688,1088L 2688,1120' >
<path id='default_workflow_data_type_fk_default_workflow_id' onmouseover="hghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" onmouseout="uhghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','qiita.default_workflow.default_workflow_id'])" transform='translate(8,0)' class='scene' d='M 2672,1088L 2672,1104' >
<title>Fk fk_default_workflow_id
default_workflow_data_type ref default_workflow ( default_workflow_id )</title>
</path>
<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2688,1088L 2688,1120' ><title>Fk fk_default_workflow_id
<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2672,1088L 2672,1104' ><title>Fk fk_default_workflow_id
default_workflow_data_type ref default_workflow ( default_workflow_id )</title></path>
<text x='2701' y='1090' transform='rotate(90 2701,1090)' class='relName'>default_workflow_id</text>
<text x='2685' y='1090' transform='rotate(90 2685,1090)' class='relName'>default_workflow_id</text>
<!-- == Fk 'default_workflow_edge_fk_default_workflow_edge' == -->
<path id='default_workflow_edge_fk_default_workflow_edge' onmouseover="hghl(['default_workflow_edge_fk_default_workflow_edge','qiita.default_workflow_edge.parent_id','qiita.default_workflow_node.default_workflow_node_id'])" onmouseout="uhghl(['default_workflow_edge_fk_default_workflow_edge','qiita.default_workflow_edge.parent_id','qiita.default_workflow_node.default_workflow_node_id'])" transform='translate(8,0)' class='scene' d='M 2640,1280L 2624,1280' >
<title>Fk fk_default_workflow_edge
Expand Down Expand Up @@ -1481,12 +1481,12 @@
<path d='M 704 1140 L 704 1120 Q 704 1112 712 1112 L 824 1112 Q 832 1112 832 1120 L 832 1140 L704 1140 ' style='fill:url(#tbg_D0DEF5);stroke:1;stroke-opacity:0.1;' />
<rect class='entity' x='704' y='1112' width='128' height='80' rx='8' ry='8' style='fill:none;stroke:#27282C'/>
<a xlink:href='#qiita.data_type'><text x='732' y='1131' >data_type</text><title>Table qiita.data_type</title></a>
<use id='nn' x='706' y='1149' xlink:href='#nn'/><use id='pk' x='706' y='1148' xlink:href='#pk'><title>Pk pk_data_type ( data_type_id ) </title></use><a xlink:href='#qiita.data_type_data_type_id'><text id='qiita.data_type.data_type_id' x='723' y='1159' onmouseover="hghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id'])" onmouseout="uhghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id'])">data_type_id</text><title>data_type_id
<use id='nn' x='706' y='1149' xlink:href='#nn'/><use id='pk' x='706' y='1148' xlink:href='#pk'><title>Pk pk_data_type ( data_type_id ) </title></use><a xlink:href='#qiita.data_type_data_type_id'><text id='qiita.data_type.data_type_id' x='723' y='1159' onmouseover="hghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id'])" onmouseout="uhghl(['analysis_filepath_fk_analysis_filepath_1','qiita.analysis_filepath.data_type_id','artifact_fk_artifact_data_type','qiita.artifact.data_type_id','default_workflow_data_type_fk_data_type_id','qiita.default_workflow_data_type.data_type_id','prep_template_fk_prep_template_data_type','qiita.prep_template.data_type_id'])">data_type_id</text><title>data_type_id
* bigserial</title></a>
<a xlink:href='#qiita.data_type.data_type_id'><use id='ref' x='820' y='1148' xlink:href='#ref'/><title>Referred by analysis_filepath ( data_type_id )
Referred by artifact ( data_type_id )
Referred by prep_template ( data_type_id )
Referred by default_workflow_data_type ( data_type_id ) </title></a>
Referred by default_workflow_data_type ( data_type_id )
Referred by prep_template ( data_type_id ) </title></a>
<use id='nn' x='706' y='1165' xlink:href='#nn'/><use id='unq' x='706' y='1164' xlink:href='#unq'><title>Unq idx_data_type ( data_type ) </title></use><a xlink:href='#qiita.data_type_data_type'><text id='qiita.data_type.data_type' x='723' y='1175'>data_type</text><title>data_type
* varchar
Data type (16S, metabolome, etc) the job will use</title></a>
Expand All @@ -1509,19 +1509,21 @@
This is a varchar here - but is of type JSON in postgresql.</title></a>
<text x='2621' y='1188' text-anchor='end' class='colType'>t</text>
<!-- == Table 'default_workflow' == -->
<rect class='entity' style='stroke:none;' x='2672' y='1128' width='176' height='96' rx='8' ry='8' />
<path d='M 2672 1156 L 2672 1136 Q 2672 1128 2680 1128 L 2840 1128 Q 2848 1128 2848 1136 L 2848 1156 L2672 1156 ' style='fill:url(#tbg_B2CDF7);stroke:1;stroke-opacity:0.1;' />
<rect class='entity' x='2672' y='1128' width='176' height='96' rx='8' ry='8' style='fill:none;stroke:#27282C'/>
<a xlink:href='#qiita.default_workflow'><text x='2701' y='1147' >default_workflow</text><title>Table qiita.default_workflow</title></a>
<use id='nn' x='2674' y='1165' xlink:href='#nn'/><use id='pk' x='2674' y='1164' xlink:href='#pk'><title>Pk pk_default_workflow ( default_workflow_id ) </title></use><a xlink:href='#qiita.default_workflow_default_workflow_id'><text id='qiita.default_workflow.default_workflow_id' x='2691' y='1175' onmouseover="hghl(['default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id','default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id'])" onmouseout="uhghl(['default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id','default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id'])">default_workflow_id</text><title>default_workflow_id
<rect class='entity' style='stroke:none;' x='2656' y='1112' width='176' height='112' rx='8' ry='8' />
<path d='M 2656 1140 L 2656 1120 Q 2656 1112 2664 1112 L 2824 1112 Q 2832 1112 2832 1120 L 2832 1140 L2656 1140 ' style='fill:url(#tbg_B2CDF7);stroke:1;stroke-opacity:0.1;' />
<rect class='entity' x='2656' y='1112' width='176' height='112' rx='8' ry='8' style='fill:none;stroke:#27282C'/>
<a xlink:href='#qiita.default_workflow'><text x='2685' y='1131' >default_workflow</text><title>Table qiita.default_workflow</title></a>
<use id='nn' x='2658' y='1149' xlink:href='#nn'/><use id='pk' x='2658' y='1148' xlink:href='#pk'><title>Pk pk_default_workflow ( default_workflow_id ) </title></use><a xlink:href='#qiita.default_workflow_default_workflow_id'><text id='qiita.default_workflow.default_workflow_id' x='2675' y='1159' onmouseover="hghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id'])" onmouseout="uhghl(['default_workflow_data_type_fk_default_workflow_id','qiita.default_workflow_data_type.default_workflow_id','default_workflow_node_fk_default_workflow_command_1','qiita.default_workflow_node.default_workflow_id'])">default_workflow_id</text><title>default_workflow_id
* bigserial</title></a>
<a xlink:href='#qiita.default_workflow.default_workflow_id'><use id='ref' x='2836' y='1164' xlink:href='#ref'/><title>Referred by default_workflow_node ( default_workflow_id )
Referred by default_workflow_data_type ( default_workflow_id ) </title></a>
<use id='nn' x='2674' y='1181' xlink:href='#nn'/><use id='unq' x='2674' y='1180' xlink:href='#unq'><title>Unq idx_default_workflow ( name ) </title></use><a xlink:href='#qiita.default_workflow_name'><text id='qiita.default_workflow.name' x='2691' y='1191'>name</text><title>name
<a xlink:href='#qiita.default_workflow.default_workflow_id'><use id='ref' x='2820' y='1148' xlink:href='#ref'/><title>Referred by default_workflow_data_type ( default_workflow_id )
Referred by default_workflow_node ( default_workflow_id ) </title></a>
<use id='nn' x='2658' y='1165' xlink:href='#nn'/><use id='unq' x='2658' y='1164' xlink:href='#unq'><title>Unq idx_default_workflow ( name ) </title></use><a xlink:href='#qiita.default_workflow_name'><text id='qiita.default_workflow.name' x='2675' y='1175'>name</text><title>name
* varchar</title></a>
<text x='2845' y='1188' text-anchor='end' class='colType'>t</text> <a xlink:href='#qiita.default_workflow_active'><text id='qiita.default_workflow.active' x='2691' y='1207'>active</text><title>active
<text x='2829' y='1172' text-anchor='end' class='colType'>t</text> <a xlink:href='#qiita.default_workflow_active'><text id='qiita.default_workflow.active' x='2675' y='1191'>active</text><title>active
boolean</title></a>
<text x='2845' y='1204' text-anchor='end' class='colType'>b</text>
<text x='2829' y='1188' text-anchor='end' class='colType'>b</text> <a xlink:href='#qiita.default_workflow_description'><text id='qiita.default_workflow.description' x='2675' y='1207'>description</text><title>description
text</title></a>
<text x='2829' y='1204' text-anchor='end' class='colType'>t</text>
<!-- == Table 'default_workflow_data_type' == -->
<rect class='entity' style='stroke:none;' x='2640' y='1000' width='208' height='80' rx='8' ry='8' />
<path d='M 2640 1028 L 2640 1008 Q 2640 1000 2648 1000 L 2840 1000 Q 2848 1000 2848 1008 L 2848 1028 L2640 1028 ' style='fill:url(#tbg_C1D8EE);stroke:1;stroke-opacity:0.1;' />
Expand Down Expand Up @@ -3696,7 +3698,7 @@
</table></div></div>

<br><br>
<div class='card'><div class='card-body'><a name='qiita.default_workflow' onclick='window.scrollTo(2472, 1136);return false;' style='cursor:pointer;'><h5 class='card-title'>Table default_workflow</h5></a>
<div class='card'><div class='card-body'><a name='qiita.default_workflow' onclick='window.scrollTo(2456, 1120);return false;' style='cursor:pointer;'><h5 class='card-title'>Table default_workflow</h5></a>
<table class='table table-sm table-bordered small' style='table-layout: fixed; word-wrap: break-word;'>
<thead>
<tr><th style='width:5%'>Idx</th><th style='width:40%'>Field Name</th><th>Data Type</th></tr>
Expand All @@ -3717,6 +3719,11 @@
<td><a name='qiita.default_workflow_active'>active</a></td>
<td> boolean </td>
</tr>
<tr>
<td>&nbsp;</td>
<td><a name='qiita.default_workflow_description'>description</a></td>
<td> text </td>
</tr>
<tr><th colspan='3'>Indexes</th></tr>
<tr> <td><svg width='14' height='14'><use xlink:href='#pk'/></svg></td><td>pk&#95;default&#95;workflow</td>
<td> ON default&#95;workflow&#95;id</td>
Expand Down
Loading