Implement export to cwl (#98)

* Implement export to cwl

* fixes

* copy workflow file

* fix

* fix path

* copy file

* last fix for tonight

* another try

* fix function name

* number all steps

* fix links

* add more CWL workflows

* more fixes

* more fixes

* black formatting

* iterate over keys

* create a cwl file for every node

* fix import of internal functions

* fix internal functions

* fix spelling

* fix function import

* fix quantum espresso

* fix nfdi

Author: jan-janssen

.github/workflows/pipeline.yml

@@ -68,6 +68,7 @@ jobs:
       run: |
         cd example_workflows/nfdi/
         papermill aiida.ipynb aiida_out.ipynb -k "python3"
+        papermill cwl.ipynb cwl_out.ipynb -k "python3"
         papermill jobflow.ipynb jobflow_out.ipynb -k "python3"
         papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3"
         papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3"
@@ -101,6 +102,7 @@ jobs:
         cp -r example_workflows/quantum_espresso/espresso .
         cd example_workflows/quantum_espresso
         papermill aiida.ipynb aiida_out.ipynb -k "python3"
+        papermill cwl.ipynb cwl_out.ipynb -k "python3"
         papermill jobflow.ipynb jobflow_out.ipynb -k "python3"
         papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3"
         papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3"
@@ -128,6 +130,7 @@ jobs:
       run: |
         cd example_workflows/arithmetic
         papermill aiida.ipynb aiida_out.ipynb -k "python3"
+        papermill cwl.ipynb cwl_out.ipynb -k "python3"
         papermill jobflow.ipynb jobflow_out.ipynb -k "python3"
         papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3"
         papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3"

binder/environment.yml

@@ -12,3 +12,4 @@ dependencies:
 - aiida-workgraph =0.5.2
 - conda_subprocess =0.0.6
 - networkx =3.4.2
+- cwltool =3.1.20250110105449

example_workflows/arithmetic/cwl.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n  sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/arithmetic/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div_0\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] /tmp/hig0p5dd$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/0xeme5n4/stgba321752-0a06-41dd-95c0-f5890f1dfaae/workflow.py \\\n    --function=workflow.get_prod_and_div \\\n    --arg_y=/tmp/0xeme5n4/stg9f21530f-2f98-44e6-80b6-afa2aac5a083/y.pickle \\\n    --arg_x=/tmp/0xeme5n4/stg2e35a78a-a776-48de-b82f-8075d3284a3c/x.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum_1\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] start\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] /tmp/u07nctrs$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/sa53wz5h/stg686b1324-0030-4bfd-bdab-64fc9ee4d491/workflow.py \\\n    --function=workflow.get_sum \\\n    --arg_y=/tmp/sa53wz5h/stg1679a702-ac89-4c84-8cb5-7a57ae9e1834/div.pickle \\\n    --arg_x=/tmp/sa53wz5h/stg6ab588fe-2723-416e-a7b8-648b1a63406e/prod.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n    \"result_file\": {\n        \"location\": \"file:///home/jovyan/example_workflows/arithmetic/result.pickle\",\n        \"basename\": \"result.pickle\",\n        \"class\": \"File\",\n        \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n        \"size\": 21,\n        \"path\": \"/home/jovyan/example_workflows/arithmetic/result.pickle\"\n    }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":4},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n    print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":5},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}

example_workflows/nfdi/cwl.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"2033dda1-dc7a-4f96-b1bd-90505b0ec555","cell_type":"code","source":"import json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"60cb8ace-acb0-47b4-b0bc-bb54d00d19dd","cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"b0cf73b9-ea21-4437-8d2a-c51b65bbfa86","cell_type":"markdown","source":"# Overwrite source directory with absolute path","metadata":{}},{"id":"bca61d32-89dd-4df7-92da-fee1a157df5a","cell_type":"code","source":"with open(\"workflow.json\") as f:\n    content = json.load(f)","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"8392fa04-4fb3-496e-9387-0106c872fb98","cell_type":"code","source":"node_lst = []\nfor n in content[\"nodes\"]:\n    if 'name' in n and n['name'] == 'source_directory':\n        n[\"value\"] = os.path.abspath(n[\"value\"])\n    node_lst.append(n)\n\ncontent[\"nodes\"] = node_lst","metadata":{"trusted":true},"outputs":[],"execution_count":6},{"id":"e53c7769-28bb-4d8e-b3a4-02298818a001","cell_type":"code","source":"with open(\"workflow.json\", \"w\") as f:\n    json.dump(content, f)","metadata":{"trusted":true},"outputs":[],"execution_count":7},{"id":"a9540ba7-f15a-4d04-86aa-0cf2ad4ac185","cell_type":"markdown","source":"# Execute workflow","metadata":{}},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":8},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool --preserve-environment=CONDA_EXE workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n  sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/nfdi/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step generate_mesh_0\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] start\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] /tmp/2zy1qyud$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/3kq1m6dv/stgfdd0e5fa-ab2c-4ee5-a621-421e0f130c2d/workflow.py \\\n    --function=workflow.generate_mesh \\\n    --arg_source_directory=/tmp/3kq1m6dv/stgb3cb8d7a-2fd4-4bed-bddd-a53347db76d4/source_directory.pickle \\\n    --arg_domain_size=/tmp/3kq1m6dv/stgf88b878d-cda3-46dc-b95b-1a5866fc79df/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] Max memory used: 51MiB\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step convert_to_xdmf_1\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] start\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] /tmp/wtfrae9y$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/mnez84gw/stg3b3a2c97-dc53-4fa4-8b26-1c0ef725455a/workflow.py \\\n    --function=workflow.convert_to_xdmf \\\n    --arg_gmsh_output_file=/tmp/mnez84gw/stg7cb20c0e-e3a4-40c7-b347-90b3861ac948/result.pickle\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] Max memory used: 62MiB\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step poisson_2\n\u001b[1;30mINFO\u001b[0m [step poisson_2] start\n\u001b[1;30mINFO\u001b[0m [job poisson_2] /tmp/c25cseik$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/mdjrrwsb/stg9fc40fe1-ad74-4530-9b90-3f25cda3228b/workflow.py \\\n    --function=workflow.poisson \\\n    --arg_source_directory=/tmp/mdjrrwsb/stg95c5865b-b3b7-41a3-9f91-8c0d4e64e2cc/source_directory.pickle \\\n    --arg_meshio_output_xdmf=/tmp/mdjrrwsb/stg3f852952-6283-4843-8a5a-799885c3a16f/xdmf_file.pickle \\\n    --arg_meshio_output_h5=/tmp/mdjrrwsb/stg4c12ec47-0d6d-4749-afb0-b44bcfdae15c/h5_file.pickle\n\u001b[1;30mINFO\u001b[0m [job poisson_2] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [step poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step plot_over_line_3\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] start\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] /tmp/6taq1l9y$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/fr8grips/stgcc344580-aae9-480d-915b-b011888e11a9/workflow.py \\\n    --function=workflow.plot_over_line \\\n    --arg_poisson_output_pvd_file=/tmp/fr8grips/stg7c2c93df-069a-4d33-8ebc-402be33d67d0/pvd_file.pickle \\\n    --arg_poisson_output_vtu_file=/tmp/fr8grips/stg481f6c15-8e7c-4198-9f1b-d61aa097b829/vtu_file.pickle \\\n    --arg_source_directory=/tmp/fr8grips/stge25af463-7689-43d8-b783-165f68b907bd/source_directory.pickle\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step substitute_macros_4\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] start\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] /tmp/ry44k1po$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/_yac4p7u/stge908c7b1-0d4d-44a4-a2d4-62c97b75d25f/workflow.py \\\n    --function=workflow.substitute_macros \\\n    --arg_ndofs=/tmp/_yac4p7u/stgd19106a7-26ec-4159-87e5-c55df4e05b3e/numdofs.pickle \\\n    --arg_source_directory=/tmp/_yac4p7u/stgea87bc3d-5042-4044-b2bb-58a83646f704/source_directory.pickle \\\n    --arg_pvbatch_output_file=/tmp/_yac4p7u/stga426ce18-a7c0-4941-9b4c-988615fb6dd6/result.pickle \\\n    --arg_domain_size=/tmp/_yac4p7u/stg75e4cce3-e411-4ffe-b1df-e5d40daa8d61/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] Max memory used: 92MiB\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step compile_paper_5\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] start\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] /tmp/u4tewx74$ python \\\n    -m \\\n    python_workflow_definition.cwl \\\n    --workflowfile=/tmp/c_jbp86a/stge99c916c-dabe-474c-b5f1-947214d3f9ed/workflow.py \\\n    --function=workflow.compile_paper \\\n    --arg_source_directory=/tmp/c_jbp86a/stg09ed111f-9743-4670-a6e6-182b1b32360d/source_directory.pickle \\\n    --arg_plot_file=/tmp/c_jbp86a/stg5649e17e-4a2b-416c-8107-31df66b6e769/result.pickle \\\n    --arg_macros_tex=/tmp/c_jbp86a/stg26421152-a968-4c63-97a0-8ed741b07f1a/result.pickle\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] Max memory used: 260MiB\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n    \"result_file\": {\n        \"location\": \"file:///home/jovyan/example_workflows/nfdi/result.pickle\",\n        \"basename\": \"result.pickle\",\n        \"class\": \"File\",\n        \"checksum\": \"sha1$1be622d292821508f2a41ec4bc8a04d06dfbdbaf\",\n        \"size\": 53,\n        \"path\": \"/home/jovyan/example_workflows/nfdi/result.pickle\"\n    }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":9},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n    print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/tmp/u4tewx74/postprocessing/paper.pdf\n"}],"execution_count":10},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}

example_workflows/quantum_espresso/cwl.ipynb

@@ -0,0 +1,51 @@
+import sys
+import pickle
+from ast import literal_eval
+import importlib.util
+
+
+def load_function(file_name, funct):
+    spec = importlib.util.spec_from_file_location("workflow", file_name)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["workflow"] = module
+    spec.loader.exec_module(module)
+    return getattr(module, funct.split(".")[-1])
+
+
+def convert_argument(arg):
+    if ".pickle" in arg:
+        with open(arg, "rb") as f:
+            return pickle.load(f)
+    else:
+        return literal_eval(arg)
+
+
+if __name__ == "__main__":
+    # load input
+    argument_lst = sys.argv[1:]
+    funct_lst = [arg.split("=")[-1] for arg in argument_lst if "--function=" in arg]
+    file_lst = [arg.split("=")[-1] for arg in argument_lst if "--workflowfile=" in arg]
+    if len(file_lst) > 0:
+        workflow_function = load_function(file_name=file_lst[0], funct=funct_lst[0])
+        internal_function = False
+    else:
+        m, p = funct_lst[0].rsplit(".", 1)
+        workflow_function = getattr(importlib.import_module(m), p)
+        internal_function = True
+    kwargs = {
+        arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1])
+        for arg in argument_lst
+        if "--arg_" in arg
+    }
+
+    # evaluate function
+    result = workflow_function(**kwargs)
+
+    # store output
+    if isinstance(result, dict) and not internal_function:
+        for k, v in result.items():
+            with open(k + ".pickle", "wb") as f:
+                pickle.dump(v, f)
+    else:
+        with open("result.pickle", "wb") as f:
+            pickle.dump(result, f)