From 54419ef13b9a564c7c5811a3f744d4aaaef83e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Thu, 15 May 2025 08:24:43 +0200 Subject: [PATCH] Move CWL functionality to init --- example_workflows/arithmetic/cwl.ipynb | 98 ++++++- example_workflows/nfdi/cwl.ipynb | 160 +++++++++++- example_workflows/quantum_espresso/cwl.ipynb | 118 ++++++++- .../cwl/__init__.py | 240 ++++++++++++++++++ .../python_workflow_definition/cwl/export.py | 240 ------------------ 5 files changed, 613 insertions(+), 243 deletions(-) delete mode 100644 python_workflow_definition/src/python_workflow_definition/cwl/export.py diff --git a/example_workflows/arithmetic/cwl.ipynb b/example_workflows/arithmetic/cwl.ipynb index 770f612..ba406e5 100644 --- a/example_workflows/arithmetic/cwl.ipynb +++ b/example_workflows/arithmetic/cwl.ipynb @@ -1 +1,97 @@ -{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/arithmetic/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div_0\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] /tmp/hig0p5dd$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/0xeme5n4/stgba321752-0a06-41dd-95c0-f5890f1dfaae/workflow.py \\\n --function=workflow.get_prod_and_div \\\n --arg_y=/tmp/0xeme5n4/stg9f21530f-2f98-44e6-80b6-afa2aac5a083/y.pickle \\\n --arg_x=/tmp/0xeme5n4/stg2e35a78a-a776-48de-b82f-8075d3284a3c/x.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum_1\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] start\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] /tmp/u07nctrs$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/sa53wz5h/stg686b1324-0030-4bfd-bdab-64fc9ee4d491/workflow.py \\\n --function=workflow.get_sum \\\n --arg_y=/tmp/sa53wz5h/stg1679a702-ac89-4c84-8cb5-7a57ae9e1834/div.pickle \\\n --arg_x=/tmp/sa53wz5h/stg6ab588fe-2723-416e-a7b8-648b1a63406e/prod.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/arithmetic/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/example_workflows/arithmetic/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":4},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":5},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file +{ + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.8", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "id": "377fef56-484d-491c-b19e-1be6931e44eb", + "cell_type": "code", + "source": "import pickle", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 1 + }, + { + "id": "92e3921b-2bb8-4333-8cfe-4bd27f785d24", + "cell_type": "code", + "source": "from python_workflow_definition.cwl import load_workflow_json", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 2 + }, + { + "id": "5303c059-8ae4-4557-858e-b4bd64eac711", + "cell_type": "code", + "source": "load_workflow_json(file_name=\"workflow.json\")", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 3 + }, + { + "id": "df302bd2-e9b6-4595-979c-67c46414d986", + "cell_type": "code", + "source": "! cwltool workflow.cwl workflow.yml", + "metadata": { + "trusted": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001B[1;30mINFO\u001B[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001B[1;30mINFO\u001B[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/arithmetic/workflow.cwl'\n\u001B[1;30mINFO\u001B[0m [workflow ] start\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_prod_and_div_0\n\u001B[1;30mINFO\u001B[0m [step get_prod_and_div_0] start\n\u001B[1;30mINFO\u001B[0m [job get_prod_and_div_0] /tmp/hig0p5dd$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/0xeme5n4/stgba321752-0a06-41dd-95c0-f5890f1dfaae/workflow.py \\\n --function=workflow.get_prod_and_div \\\n --arg_y=/tmp/0xeme5n4/stg9f21530f-2f98-44e6-80b6-afa2aac5a083/y.pickle \\\n --arg_x=/tmp/0xeme5n4/stg2e35a78a-a776-48de-b82f-8075d3284a3c/x.pickle\n\u001B[1;30mINFO\u001B[0m [job get_prod_and_div_0] completed success\n\u001B[1;30mINFO\u001B[0m [step get_prod_and_div_0] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_sum_1\n\u001B[1;30mINFO\u001B[0m [step get_sum_1] start\n\u001B[1;30mINFO\u001B[0m [job get_sum_1] /tmp/u07nctrs$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/sa53wz5h/stg686b1324-0030-4bfd-bdab-64fc9ee4d491/workflow.py \\\n --function=workflow.get_sum \\\n --arg_y=/tmp/sa53wz5h/stg1679a702-ac89-4c84-8cb5-7a57ae9e1834/div.pickle \\\n --arg_x=/tmp/sa53wz5h/stg6ab588fe-2723-416e-a7b8-648b1a63406e/prod.pickle\n\u001B[1;30mINFO\u001B[0m [job get_sum_1] completed success\n\u001B[1;30mINFO\u001B[0m [step get_sum_1] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/arithmetic/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/example_workflows/arithmetic/result.pickle\"\n }\n}\u001B[1;30mINFO\u001B[0m Final process status is success\n" + } + ], + "execution_count": 4 + }, + { + "id": "2942dbba-ea0a-4d20-be5c-ed9992d09ff8", + "cell_type": "code", + "source": "with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))", + "metadata": { + "trusted": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "2.5\n" + } + ], + "execution_count": 5 + }, + { + "id": "60e909ee-d0d0-4bd1-81c8-dd5274ae5834", + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + } + ] +} diff --git a/example_workflows/nfdi/cwl.ipynb b/example_workflows/nfdi/cwl.ipynb index f3d625a..40dde74 100644 --- a/example_workflows/nfdi/cwl.ipynb +++ b/example_workflows/nfdi/cwl.ipynb @@ -1 +1,159 @@ -{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"2033dda1-dc7a-4f96-b1bd-90505b0ec555","cell_type":"code","source":"import json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"60cb8ace-acb0-47b4-b0bc-bb54d00d19dd","cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"b0cf73b9-ea21-4437-8d2a-c51b65bbfa86","cell_type":"markdown","source":"# Overwrite source directory with absolute path","metadata":{}},{"id":"bca61d32-89dd-4df7-92da-fee1a157df5a","cell_type":"code","source":"with open(\"workflow.json\") as f:\n content = json.load(f)","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"8392fa04-4fb3-496e-9387-0106c872fb98","cell_type":"code","source":"node_lst = []\nfor n in content[\"nodes\"]:\n if 'name' in n and n['name'] == 'source_directory':\n n[\"value\"] = os.path.abspath(n[\"value\"])\n node_lst.append(n)\n\ncontent[\"nodes\"] = node_lst","metadata":{"trusted":true},"outputs":[],"execution_count":6},{"id":"e53c7769-28bb-4d8e-b3a4-02298818a001","cell_type":"code","source":"with open(\"workflow.json\", \"w\") as f:\n json.dump(content, f)","metadata":{"trusted":true},"outputs":[],"execution_count":7},{"id":"a9540ba7-f15a-4d04-86aa-0cf2ad4ac185","cell_type":"markdown","source":"# Execute workflow","metadata":{}},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":8},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool --preserve-environment=CONDA_EXE workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/nfdi/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step generate_mesh_0\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] start\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] /tmp/2zy1qyud$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3kq1m6dv/stgfdd0e5fa-ab2c-4ee5-a621-421e0f130c2d/workflow.py \\\n --function=workflow.generate_mesh \\\n --arg_source_directory=/tmp/3kq1m6dv/stgb3cb8d7a-2fd4-4bed-bddd-a53347db76d4/source_directory.pickle \\\n --arg_domain_size=/tmp/3kq1m6dv/stgf88b878d-cda3-46dc-b95b-1a5866fc79df/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] Max memory used: 51MiB\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step convert_to_xdmf_1\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] start\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] /tmp/wtfrae9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mnez84gw/stg3b3a2c97-dc53-4fa4-8b26-1c0ef725455a/workflow.py \\\n --function=workflow.convert_to_xdmf \\\n --arg_gmsh_output_file=/tmp/mnez84gw/stg7cb20c0e-e3a4-40c7-b347-90b3861ac948/result.pickle\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] Max memory used: 62MiB\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step poisson_2\n\u001b[1;30mINFO\u001b[0m [step poisson_2] start\n\u001b[1;30mINFO\u001b[0m [job poisson_2] /tmp/c25cseik$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mdjrrwsb/stg9fc40fe1-ad74-4530-9b90-3f25cda3228b/workflow.py \\\n --function=workflow.poisson \\\n --arg_source_directory=/tmp/mdjrrwsb/stg95c5865b-b3b7-41a3-9f91-8c0d4e64e2cc/source_directory.pickle \\\n --arg_meshio_output_xdmf=/tmp/mdjrrwsb/stg3f852952-6283-4843-8a5a-799885c3a16f/xdmf_file.pickle \\\n --arg_meshio_output_h5=/tmp/mdjrrwsb/stg4c12ec47-0d6d-4749-afb0-b44bcfdae15c/h5_file.pickle\n\u001b[1;30mINFO\u001b[0m [job poisson_2] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [step poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step plot_over_line_3\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] start\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] /tmp/6taq1l9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/fr8grips/stgcc344580-aae9-480d-915b-b011888e11a9/workflow.py \\\n --function=workflow.plot_over_line \\\n --arg_poisson_output_pvd_file=/tmp/fr8grips/stg7c2c93df-069a-4d33-8ebc-402be33d67d0/pvd_file.pickle \\\n --arg_poisson_output_vtu_file=/tmp/fr8grips/stg481f6c15-8e7c-4198-9f1b-d61aa097b829/vtu_file.pickle \\\n --arg_source_directory=/tmp/fr8grips/stge25af463-7689-43d8-b783-165f68b907bd/source_directory.pickle\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step substitute_macros_4\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] start\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] /tmp/ry44k1po$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/_yac4p7u/stge908c7b1-0d4d-44a4-a2d4-62c97b75d25f/workflow.py \\\n --function=workflow.substitute_macros \\\n --arg_ndofs=/tmp/_yac4p7u/stgd19106a7-26ec-4159-87e5-c55df4e05b3e/numdofs.pickle \\\n --arg_source_directory=/tmp/_yac4p7u/stgea87bc3d-5042-4044-b2bb-58a83646f704/source_directory.pickle \\\n --arg_pvbatch_output_file=/tmp/_yac4p7u/stga426ce18-a7c0-4941-9b4c-988615fb6dd6/result.pickle \\\n --arg_domain_size=/tmp/_yac4p7u/stg75e4cce3-e411-4ffe-b1df-e5d40daa8d61/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] Max memory used: 92MiB\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step compile_paper_5\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] start\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] /tmp/u4tewx74$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/c_jbp86a/stge99c916c-dabe-474c-b5f1-947214d3f9ed/workflow.py \\\n --function=workflow.compile_paper \\\n --arg_source_directory=/tmp/c_jbp86a/stg09ed111f-9743-4670-a6e6-182b1b32360d/source_directory.pickle \\\n --arg_plot_file=/tmp/c_jbp86a/stg5649e17e-4a2b-416c-8107-31df66b6e769/result.pickle \\\n --arg_macros_tex=/tmp/c_jbp86a/stg26421152-a968-4c63-97a0-8ed741b07f1a/result.pickle\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] Max memory used: 260MiB\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/nfdi/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$1be622d292821508f2a41ec4bc8a04d06dfbdbaf\",\n \"size\": 53,\n \"path\": \"/home/jovyan/example_workflows/nfdi/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":9},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/tmp/u4tewx74/postprocessing/paper.pdf\n"}],"execution_count":10},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file +{ + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.8", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "id": "377fef56-484d-491c-b19e-1be6931e44eb", + "cell_type": "code", + "source": "import pickle", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 1 + }, + { + "id": "2033dda1-dc7a-4f96-b1bd-90505b0ec555", + "cell_type": "code", + "source": "import json", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 2 + }, + { + "id": "60cb8ace-acb0-47b4-b0bc-bb54d00d19dd", + "cell_type": "code", + "source": "import os", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 3 + }, + { + "id": "92e3921b-2bb8-4333-8cfe-4bd27f785d24", + "cell_type": "code", + "source": "from python_workflow_definition.cwl import load_workflow_json", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 4 + }, + { + "id": "b0cf73b9-ea21-4437-8d2a-c51b65bbfa86", + "cell_type": "markdown", + "source": "# Overwrite source directory with absolute path", + "metadata": {} + }, + { + "id": "bca61d32-89dd-4df7-92da-fee1a157df5a", + "cell_type": "code", + "source": "with open(\"workflow.json\") as f:\n content = json.load(f)", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 5 + }, + { + "id": "8392fa04-4fb3-496e-9387-0106c872fb98", + "cell_type": "code", + "source": "node_lst = []\nfor n in content[\"nodes\"]:\n if 'name' in n and n['name'] == 'source_directory':\n n[\"value\"] = os.path.abspath(n[\"value\"])\n node_lst.append(n)\n\ncontent[\"nodes\"] = node_lst", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 6 + }, + { + "id": "e53c7769-28bb-4d8e-b3a4-02298818a001", + "cell_type": "code", + "source": "with open(\"workflow.json\", \"w\") as f:\n json.dump(content, f)", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 7 + }, + { + "id": "a9540ba7-f15a-4d04-86aa-0cf2ad4ac185", + "cell_type": "markdown", + "source": "# Execute workflow", + "metadata": {} + }, + { + "id": "5303c059-8ae4-4557-858e-b4bd64eac711", + "cell_type": "code", + "source": "load_workflow_json(file_name=\"workflow.json\")", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 8 + }, + { + "id": "df302bd2-e9b6-4595-979c-67c46414d986", + "cell_type": "code", + "source": "! cwltool --preserve-environment=CONDA_EXE workflow.cwl workflow.yml", + "metadata": { + "trusted": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001B[1;30mINFO\u001B[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001B[1;30mINFO\u001B[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/nfdi/workflow.cwl'\n\u001B[1;30mINFO\u001B[0m [workflow ] start\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step generate_mesh_0\n\u001B[1;30mINFO\u001B[0m [step generate_mesh_0] start\n\u001B[1;30mINFO\u001B[0m [job generate_mesh_0] /tmp/2zy1qyud$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3kq1m6dv/stgfdd0e5fa-ab2c-4ee5-a621-421e0f130c2d/workflow.py \\\n --function=workflow.generate_mesh \\\n --arg_source_directory=/tmp/3kq1m6dv/stgb3cb8d7a-2fd4-4bed-bddd-a53347db76d4/source_directory.pickle \\\n --arg_domain_size=/tmp/3kq1m6dv/stgf88b878d-cda3-46dc-b95b-1a5866fc79df/domain_size.pickle\n\u001B[1;30mINFO\u001B[0m [job generate_mesh_0] Max memory used: 51MiB\n\u001B[1;30mINFO\u001B[0m [job generate_mesh_0] completed success\n\u001B[1;30mINFO\u001B[0m [step generate_mesh_0] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step convert_to_xdmf_1\n\u001B[1;30mINFO\u001B[0m [step convert_to_xdmf_1] start\n\u001B[1;30mINFO\u001B[0m [job convert_to_xdmf_1] /tmp/wtfrae9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mnez84gw/stg3b3a2c97-dc53-4fa4-8b26-1c0ef725455a/workflow.py \\\n --function=workflow.convert_to_xdmf \\\n --arg_gmsh_output_file=/tmp/mnez84gw/stg7cb20c0e-e3a4-40c7-b347-90b3861ac948/result.pickle\n\u001B[1;30mINFO\u001B[0m [job convert_to_xdmf_1] Max memory used: 62MiB\n\u001B[1;30mINFO\u001B[0m [job convert_to_xdmf_1] completed success\n\u001B[1;30mINFO\u001B[0m [step convert_to_xdmf_1] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step poisson_2\n\u001B[1;30mINFO\u001B[0m [step poisson_2] start\n\u001B[1;30mINFO\u001B[0m [job poisson_2] /tmp/c25cseik$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mdjrrwsb/stg9fc40fe1-ad74-4530-9b90-3f25cda3228b/workflow.py \\\n --function=workflow.poisson \\\n --arg_source_directory=/tmp/mdjrrwsb/stg95c5865b-b3b7-41a3-9f91-8c0d4e64e2cc/source_directory.pickle \\\n --arg_meshio_output_xdmf=/tmp/mdjrrwsb/stg3f852952-6283-4843-8a5a-799885c3a16f/xdmf_file.pickle \\\n --arg_meshio_output_h5=/tmp/mdjrrwsb/stg4c12ec47-0d6d-4749-afb0-b44bcfdae15c/h5_file.pickle\n\u001B[1;30mINFO\u001B[0m [job poisson_2] Max memory used: 71MiB\n\u001B[1;30mINFO\u001B[0m [job poisson_2] completed success\n\u001B[1;30mINFO\u001B[0m [step poisson_2] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step plot_over_line_3\n\u001B[1;30mINFO\u001B[0m [step plot_over_line_3] start\n\u001B[1;30mINFO\u001B[0m [job plot_over_line_3] /tmp/6taq1l9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/fr8grips/stgcc344580-aae9-480d-915b-b011888e11a9/workflow.py \\\n --function=workflow.plot_over_line \\\n --arg_poisson_output_pvd_file=/tmp/fr8grips/stg7c2c93df-069a-4d33-8ebc-402be33d67d0/pvd_file.pickle \\\n --arg_poisson_output_vtu_file=/tmp/fr8grips/stg481f6c15-8e7c-4198-9f1b-d61aa097b829/vtu_file.pickle \\\n --arg_source_directory=/tmp/fr8grips/stge25af463-7689-43d8-b783-165f68b907bd/source_directory.pickle\n\u001B[1;30mINFO\u001B[0m [job plot_over_line_3] Max memory used: 71MiB\n\u001B[1;30mINFO\u001B[0m [job plot_over_line_3] completed success\n\u001B[1;30mINFO\u001B[0m [step plot_over_line_3] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step substitute_macros_4\n\u001B[1;30mINFO\u001B[0m [step substitute_macros_4] start\n\u001B[1;30mINFO\u001B[0m [job substitute_macros_4] /tmp/ry44k1po$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/_yac4p7u/stge908c7b1-0d4d-44a4-a2d4-62c97b75d25f/workflow.py \\\n --function=workflow.substitute_macros \\\n --arg_ndofs=/tmp/_yac4p7u/stgd19106a7-26ec-4159-87e5-c55df4e05b3e/numdofs.pickle \\\n --arg_source_directory=/tmp/_yac4p7u/stgea87bc3d-5042-4044-b2bb-58a83646f704/source_directory.pickle \\\n --arg_pvbatch_output_file=/tmp/_yac4p7u/stga426ce18-a7c0-4941-9b4c-988615fb6dd6/result.pickle \\\n --arg_domain_size=/tmp/_yac4p7u/stg75e4cce3-e411-4ffe-b1df-e5d40daa8d61/domain_size.pickle\n\u001B[1;30mINFO\u001B[0m [job substitute_macros_4] Max memory used: 92MiB\n\u001B[1;30mINFO\u001B[0m [job substitute_macros_4] completed success\n\u001B[1;30mINFO\u001B[0m [step substitute_macros_4] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step compile_paper_5\n\u001B[1;30mINFO\u001B[0m [step compile_paper_5] start\n\u001B[1;30mINFO\u001B[0m [job compile_paper_5] /tmp/u4tewx74$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/c_jbp86a/stge99c916c-dabe-474c-b5f1-947214d3f9ed/workflow.py \\\n --function=workflow.compile_paper \\\n --arg_source_directory=/tmp/c_jbp86a/stg09ed111f-9743-4670-a6e6-182b1b32360d/source_directory.pickle \\\n --arg_plot_file=/tmp/c_jbp86a/stg5649e17e-4a2b-416c-8107-31df66b6e769/result.pickle \\\n --arg_macros_tex=/tmp/c_jbp86a/stg26421152-a968-4c63-97a0-8ed741b07f1a/result.pickle\n\u001B[1;30mINFO\u001B[0m [job compile_paper_5] Max memory used: 260MiB\n\u001B[1;30mINFO\u001B[0m [job compile_paper_5] completed success\n\u001B[1;30mINFO\u001B[0m [step compile_paper_5] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/nfdi/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$1be622d292821508f2a41ec4bc8a04d06dfbdbaf\",\n \"size\": 53,\n \"path\": \"/home/jovyan/example_workflows/nfdi/result.pickle\"\n }\n}\u001B[1;30mINFO\u001B[0m Final process status is success\n" + } + ], + "execution_count": 9 + }, + { + "id": "2942dbba-ea0a-4d20-be5c-ed9992d09ff8", + "cell_type": "code", + "source": "with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))", + "metadata": { + "trusted": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "/tmp/u4tewx74/postprocessing/paper.pdf\n" + } + ], + "execution_count": 10 + }, + { + "id": "60e909ee-d0d0-4bd1-81c8-dd5274ae5834", + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + } + ] +} diff --git a/example_workflows/quantum_espresso/cwl.ipynb b/example_workflows/quantum_espresso/cwl.ipynb index da2fcbc..794600d 100644 --- a/example_workflows/quantum_espresso/cwl.ipynb +++ b/example_workflows/quantum_espresso/cwl.ipynb @@ -1 +1,117 @@ -{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"4eca79ef-1053-4f69-89ad-2bee8411068e","cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"0192ca74-3971-464b-9435-c156e0b6e623","cell_type":"code","source":"os.environ[\"ESPRESSO_PSEUDO\"] = os.path.abspath(os.path.join(\"../../espresso/pseudo\"))","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool --preserve-environment=ESPRESSO_PSEUDO workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/quantum_espresso/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_bulk_structure_0\n\u001b[1;30mINFO\u001b[0m [step get_bulk_structure_0] start\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] /tmp/o5wvbmxj$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3uzk268a/stgc8824dc9-f08d-44b9-9d9c-518dc0415f4d/workflow.py \\\n --function=workflow.get_bulk_structure \\\n --arg_a=/tmp/3uzk268a/stgbcb5cab3-e9ef-46e5-8f20-1a73f5b242e9/a.pickle \\\n --arg_element=/tmp/3uzk268a/stgb2678406-1178-4bf6-ba96-2b623c74f408/element.pickle \\\n --arg_cubic=/tmp/3uzk268a/stg7d836c8f-b499-4ea6-a341-97ff1a366b8f/cubic.pickle\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] Max memory used: 100MiB\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] completed success\n\u001b[1;30mINFO\u001b[0m [step get_bulk_structure_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_13\n\u001b[1;30mINFO\u001b[0m [step get_dict_13] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_13] /tmp/que74c90$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/k_sm4nc9/stg029d45cb-9d3e-4d26-a4f6-95cb0fb162d5/kpts.pickle \\\n --arg_structure=/tmp/k_sm4nc9/stgbde5b9c0-6926-4c08-886c-a7425fdf5571/result.pickle \\\n --arg_pseudopotentials=/tmp/k_sm4nc9/stg6717dca8-3880-4d1b-bed2-4078dca65fdb/pseudopotentials.pickle \\\n --arg_calculation=/tmp/k_sm4nc9/stg0e219c86-340f-4f1a-bc25-e366286137ea/calculation_0.pickle \\\n --arg_smearing=/tmp/k_sm4nc9/stg299bfaba-cb42-4aa5-b372-3101efa15660/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_13] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_13] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_1\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_1] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] /tmp/9ew4474v$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3hzmn68a/stgf58c4a14-7b0f-4efc-bedd-3561b9fdb227/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3hzmn68a/stg41392280-cfd6-4e67-ab11-915bfcbd0304/result.pickle \\\n --arg_working_directory=/tmp/3hzmn68a/stg6aa2aff2-eca4-41b7-bef5-a917188e0853/working_directory_0.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01664] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] Max memory used: 250MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step generate_structures_2\n\u001b[1;30mINFO\u001b[0m [step generate_structures_2] start\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] /tmp/l8wkgfu9$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/aibj9rr4/stgb715e49b-e705-4371-8c63-5bc9f8addb97/workflow.py \\\n --function=workflow.generate_structures \\\n --arg_structure=/tmp/aibj9rr4/stg88fe1a91-5cb6-4f9c-9fff-a63e477a0ccb/structure.pickle \\\n --arg_strain_lst=/tmp/aibj9rr4/stg3489de05-cdc1-4f7f-af63-cb8612cefe10/strain_lst.pickle\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] Max memory used: 108MiB\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] completed success\n\u001b[1;30mINFO\u001b[0m [step generate_structures_2] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_20\n\u001b[1;30mINFO\u001b[0m [step get_dict_20] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_20] /tmp/0ls08hbo$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/yb9nqwe7/stg475a5983-0e88-40ee-b3a4-0c9b916f08f1/kpts.pickle \\\n --arg_structure=/tmp/yb9nqwe7/stg4f27a44f-d76c-4278-855d-ab654ebe6e32/s_0.pickle \\\n --arg_pseudopotentials=/tmp/yb9nqwe7/stgbfcb2c02-776a-4101-b906-06a8ef758140/pseudopotentials.pickle \\\n --arg_calculation=/tmp/yb9nqwe7/stgd91d742a-82f8-459e-939b-2cd0876f41ee/calculation_1.pickle \\\n --arg_smearing=/tmp/yb9nqwe7/stga65643a5-b1bb-416e-bc6c-ee7b52c28bd8/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_20] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_20] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_3\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_3] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] /tmp/nzsw1xr6$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/gjwsgfmj/stg3069d789-517f-41bf-8d4b-07e51f9e924d/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/gjwsgfmj/stg9575b9dd-3336-4e26-a106-7c52c1e41f91/result.pickle \\\n --arg_working_directory=/tmp/gjwsgfmj/stg28ff68ea-caa6-47bf-be5e-67394ab72f64/working_directory_1.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01846] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] Max memory used: 239MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_3] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_25\n\u001b[1;30mINFO\u001b[0m [step get_dict_25] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_25] /tmp/zrs68r3d$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/jj2_r82j/stg13ba3696-061f-46f7-b012-bb7ab8a40796/kpts.pickle \\\n --arg_structure=/tmp/jj2_r82j/stg5ca9e71a-98fe-44df-8ca5-550908525a4e/s_2.pickle \\\n --arg_pseudopotentials=/tmp/jj2_r82j/stg9e1fe3f0-1192-4cde-9772-6be8dfa314fe/pseudopotentials.pickle \\\n --arg_calculation=/tmp/jj2_r82j/stgde664722-b13c-4ae9-9562-aec2063d548a/calculation_1.pickle \\\n --arg_smearing=/tmp/jj2_r82j/stg799e1f26-f08e-40bf-a7a9-c58e907674c8/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_25] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_25] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_23\n\u001b[1;30mINFO\u001b[0m [step get_dict_23] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_23] /tmp/7qgifyu_$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/ojouuynn/stg7b99ab8b-9235-4007-b6b7-dd4170bd1760/kpts.pickle \\\n --arg_structure=/tmp/ojouuynn/stg8bfd87ff-e83d-4918-9a0d-71b958c46154/s_1.pickle \\\n --arg_pseudopotentials=/tmp/ojouuynn/stg0bcc3451-1700-46e0-9a5e-fca9968c8521/pseudopotentials.pickle \\\n --arg_calculation=/tmp/ojouuynn/stgb9d8de26-7d88-4119-ad13-8374ff498f72/calculation_1.pickle \\\n --arg_smearing=/tmp/ojouuynn/stg01fe91f8-b588-4adc-b207-9d9d4e0956e3/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_23] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_23] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_27\n\u001b[1;30mINFO\u001b[0m [step get_dict_27] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_27] /tmp/v83aah99$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/_wg2md43/stgee67f0e5-87a9-4768-82ed-cb6c6cf74b6d/kpts.pickle \\\n --arg_structure=/tmp/_wg2md43/stgb39aa4d9-0909-4ffb-bfe8-c448a3d77669/s_3.pickle \\\n --arg_pseudopotentials=/tmp/_wg2md43/stgc7b618ee-6e90-4b93-a079-785fab087237/pseudopotentials.pickle \\\n --arg_calculation=/tmp/_wg2md43/stg738933fd-9a31-4c0e-ac99-984d1002fd0a/calculation_1.pickle \\\n --arg_smearing=/tmp/_wg2md43/stg174b61ec-1446-4c63-a433-49a940e48f39/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_27] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_27] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_4\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_4] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] /tmp/abio0491$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/pltanpun/stg312b10e4-f5f2-4770-b776-12c659453bef/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/pltanpun/stgceefbf38-ae57-4369-872d-4ea0895a6976/result.pickle \\\n --arg_working_directory=/tmp/pltanpun/stg6abc0b8b-ed0d-4692-b100-b510fe7de457/working_directory_2.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01928] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] Max memory used: 244MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_4] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_29\n\u001b[1;30mINFO\u001b[0m [step get_dict_29] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_29] /tmp/li__ygke$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/20wxb6mt/stg4f9c6f04-e0da-4f1d-9b91-17867111fcec/kpts.pickle \\\n --arg_structure=/tmp/20wxb6mt/stg27aeb042-5dba-406d-ad3f-c1a15414d462/s_4.pickle \\\n --arg_pseudopotentials=/tmp/20wxb6mt/stg00a521b7-0d6f-44fa-849f-611e82b816d2/pseudopotentials.pickle \\\n --arg_calculation=/tmp/20wxb6mt/stg51a4e465-9a6d-465e-a329-ec65c9631dca/calculation_1.pickle \\\n --arg_smearing=/tmp/20wxb6mt/stg72f4b40d-8647-441e-94ce-b4d60eb84837/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_29] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_29] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_7\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_7] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] /tmp/26ontyw1$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3e7bw8e2/stg86169764-ce4e-40f6-9b63-ef5e11208258/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3e7bw8e2/stge349561c-3f24-48da-94fa-2d16e4df4779/result.pickle \\\n --arg_working_directory=/tmp/3e7bw8e2/stg30622d6d-1c64-4440-9d76-7a70e081ff5e/working_directory_5.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02006] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] Max memory used: 243MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_7] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_5\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_5] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] /tmp/nwlrltkk$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/q_jjx0ja/stgb8b85eae-791f-4e34-9696-ceb277f72155/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/q_jjx0ja/stg2efa8335-1138-47c3-b9e9-d8ea2d8754c0/result.pickle \\\n --arg_working_directory=/tmp/q_jjx0ja/stg4f726c25-7adb-4ce0-970e-bd9ddc5191cc/working_directory_3.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02086] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] Max memory used: 246MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_5] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_6\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_6] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] /tmp/9yp55i6u$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/1c_enl4x/stg2aeb7a80-3364-48a0-810d-955e246aa33b/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/1c_enl4x/stg19bec723-020c-4d0e-bca0-8124f6962c7e/result.pickle \\\n --arg_working_directory=/tmp/1c_enl4x/stgc36f93b3-f6f6-4fec-8eb2-0de1ca22f3cd/working_directory_4.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02164] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] Max memory used: 243MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_6] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_list_31\n\u001b[1;30mINFO\u001b[0m [step get_list_31] start\n\u001b[1;30mINFO\u001b[0m [job get_list_31] /tmp/xgme48c0$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/u2adengu/stg036d73c4-50fd-4b84-9054-ef00e66e0ea8/energy.pickle \\\n --arg_4=/tmp/u2adengu/stg8736d42d-b110-43b3-8e1d-da22b9883ff5/energy.pickle \\\n --arg_1=/tmp/u2adengu/stg8cac7eb7-6132-4085-865d-673027e26baa/energy.pickle \\\n --arg_3=/tmp/u2adengu/stg55b370b4-c852-4493-bb7b-4acaa0588aa8/energy.pickle \\\n --arg_0=/tmp/u2adengu/stg5d319d67-b4fc-4c4d-886a-529b9bfd2458/energy.pickle\n\u001b[1;30mINFO\u001b[0m [job get_list_31] completed success\n\u001b[1;30mINFO\u001b[0m [step get_list_31] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_list_30\n\u001b[1;30mINFO\u001b[0m [step get_list_30] start\n\u001b[1;30mINFO\u001b[0m [job get_list_30] /tmp/1xbu0rii$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/0ads3xzd/stg24fc8770-2739-45d1-b535-89924a6234ed/volume.pickle \\\n --arg_4=/tmp/0ads3xzd/stgfb79ea5f-6109-4f2e-a1d6-0817648a5350/volume.pickle \\\n --arg_1=/tmp/0ads3xzd/stg0b1e9637-dba2-4e42-9dbe-c605e7469714/volume.pickle \\\n --arg_3=/tmp/0ads3xzd/stgdf4991ae-4266-4ab6-82be-8f87d46fa7f6/volume.pickle \\\n --arg_0=/tmp/0ads3xzd/stg623a06b5-d2a0-44fd-9473-833452ad5762/volume.pickle\n\u001b[1;30mINFO\u001b[0m [job get_list_30] completed success\n\u001b[1;30mINFO\u001b[0m [step get_list_30] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step plot_energy_volume_curve_8\n\u001b[1;30mINFO\u001b[0m [step plot_energy_volume_curve_8] start\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] /tmp/vcffw_nr$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/6s32h5zf/stg79d5980d-139d-4481-9277-3f58bb27d49f/workflow.py \\\n --function=workflow.plot_energy_volume_curve \\\n --arg_volume_lst=/tmp/6s32h5zf/stg67eb8ed0-f498-4181-a5a5-c7e74ecd3276/result.pickle \\\n --arg_energy_lst=/tmp/6s32h5zf/stg3ebdb224-bd9a-4ae7-ad4e-03489dcfb661/result.pickle\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] Max memory used: 110MiB\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] completed success\n\u001b[1;30mINFO\u001b[0m [step plot_energy_volume_curve_8] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/quantum_espresso/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$dbc1aaddc8b7343d6d33b34edcf608b8f8801918\",\n \"size\": 4,\n \"path\": \"/home/jovyan/example_workflows/quantum_espresso/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":6},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"None\n"}],"execution_count":7},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file +{ + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.8", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "id": "4eca79ef-1053-4f69-89ad-2bee8411068e", + "cell_type": "code", + "source": "import os", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 1 + }, + { + "id": "377fef56-484d-491c-b19e-1be6931e44eb", + "cell_type": "code", + "source": "import pickle", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 2 + }, + { + "id": "92e3921b-2bb8-4333-8cfe-4bd27f785d24", + "cell_type": "code", + "source": "from python_workflow_definition.cwl import load_workflow_json", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 3 + }, + { + "id": "5303c059-8ae4-4557-858e-b4bd64eac711", + "cell_type": "code", + "source": "load_workflow_json(file_name=\"workflow.json\")", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 4 + }, + { + "id": "0192ca74-3971-464b-9435-c156e0b6e623", + "cell_type": "code", + "source": "os.environ[\"ESPRESSO_PSEUDO\"] = os.path.abspath(os.path.join(\"../../espresso/pseudo\"))", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": 5 + }, + { + "id": "df302bd2-e9b6-4595-979c-67c46414d986", + "cell_type": "code", + "source": "! cwltool --preserve-environment=ESPRESSO_PSEUDO workflow.cwl workflow.yml", + "metadata": { + "trusted": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001B[1;30mINFO\u001B[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001B[1;30mINFO\u001B[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/quantum_espresso/workflow.cwl'\n\u001B[1;30mINFO\u001B[0m [workflow ] start\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_bulk_structure_0\n\u001B[1;30mINFO\u001B[0m [step get_bulk_structure_0] start\n\u001B[1;30mINFO\u001B[0m [job get_bulk_structure_0] /tmp/o5wvbmxj$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3uzk268a/stgc8824dc9-f08d-44b9-9d9c-518dc0415f4d/workflow.py \\\n --function=workflow.get_bulk_structure \\\n --arg_a=/tmp/3uzk268a/stgbcb5cab3-e9ef-46e5-8f20-1a73f5b242e9/a.pickle \\\n --arg_element=/tmp/3uzk268a/stgb2678406-1178-4bf6-ba96-2b623c74f408/element.pickle \\\n --arg_cubic=/tmp/3uzk268a/stg7d836c8f-b499-4ea6-a341-97ff1a366b8f/cubic.pickle\n\u001B[1;30mINFO\u001B[0m [job get_bulk_structure_0] Max memory used: 100MiB\n\u001B[1;30mINFO\u001B[0m [job get_bulk_structure_0] completed success\n\u001B[1;30mINFO\u001B[0m [step get_bulk_structure_0] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_dict_13\n\u001B[1;30mINFO\u001B[0m [step get_dict_13] start\n\u001B[1;30mINFO\u001B[0m [job get_dict_13] /tmp/que74c90$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/k_sm4nc9/stg029d45cb-9d3e-4d26-a4f6-95cb0fb162d5/kpts.pickle \\\n --arg_structure=/tmp/k_sm4nc9/stgbde5b9c0-6926-4c08-886c-a7425fdf5571/result.pickle \\\n --arg_pseudopotentials=/tmp/k_sm4nc9/stg6717dca8-3880-4d1b-bed2-4078dca65fdb/pseudopotentials.pickle \\\n --arg_calculation=/tmp/k_sm4nc9/stg0e219c86-340f-4f1a-bc25-e366286137ea/calculation_0.pickle \\\n --arg_smearing=/tmp/k_sm4nc9/stg299bfaba-cb42-4aa5-b372-3101efa15660/smearing.pickle\n\u001B[1;30mINFO\u001B[0m [job get_dict_13] completed success\n\u001B[1;30mINFO\u001B[0m [step get_dict_13] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step calculate_qe_1\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_1] start\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_1] /tmp/9ew4474v$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3hzmn68a/stgf58c4a14-7b0f-4efc-bedd-3561b9fdb227/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3hzmn68a/stg41392280-cfd6-4e67-ab11-915bfcbd0304/result.pickle \\\n --arg_working_directory=/tmp/3hzmn68a/stg6aa2aff2-eca4-41b7-bef5-a917188e0853/working_directory_0.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01664] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_1] Max memory used: 250MiB\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_1] completed success\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_1] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step generate_structures_2\n\u001B[1;30mINFO\u001B[0m [step generate_structures_2] start\n\u001B[1;30mINFO\u001B[0m [job generate_structures_2] /tmp/l8wkgfu9$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/aibj9rr4/stgb715e49b-e705-4371-8c63-5bc9f8addb97/workflow.py \\\n --function=workflow.generate_structures \\\n --arg_structure=/tmp/aibj9rr4/stg88fe1a91-5cb6-4f9c-9fff-a63e477a0ccb/structure.pickle \\\n --arg_strain_lst=/tmp/aibj9rr4/stg3489de05-cdc1-4f7f-af63-cb8612cefe10/strain_lst.pickle\n\u001B[1;30mINFO\u001B[0m [job generate_structures_2] Max memory used: 108MiB\n\u001B[1;30mINFO\u001B[0m [job generate_structures_2] completed success\n\u001B[1;30mINFO\u001B[0m [step generate_structures_2] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_dict_20\n\u001B[1;30mINFO\u001B[0m [step get_dict_20] start\n\u001B[1;30mINFO\u001B[0m [job get_dict_20] /tmp/0ls08hbo$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/yb9nqwe7/stg475a5983-0e88-40ee-b3a4-0c9b916f08f1/kpts.pickle \\\n --arg_structure=/tmp/yb9nqwe7/stg4f27a44f-d76c-4278-855d-ab654ebe6e32/s_0.pickle \\\n --arg_pseudopotentials=/tmp/yb9nqwe7/stgbfcb2c02-776a-4101-b906-06a8ef758140/pseudopotentials.pickle \\\n --arg_calculation=/tmp/yb9nqwe7/stgd91d742a-82f8-459e-939b-2cd0876f41ee/calculation_1.pickle \\\n --arg_smearing=/tmp/yb9nqwe7/stga65643a5-b1bb-416e-bc6c-ee7b52c28bd8/smearing.pickle\n\u001B[1;30mINFO\u001B[0m [job get_dict_20] completed success\n\u001B[1;30mINFO\u001B[0m [step get_dict_20] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step calculate_qe_3\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_3] start\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_3] /tmp/nzsw1xr6$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/gjwsgfmj/stg3069d789-517f-41bf-8d4b-07e51f9e924d/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/gjwsgfmj/stg9575b9dd-3336-4e26-a106-7c52c1e41f91/result.pickle \\\n --arg_working_directory=/tmp/gjwsgfmj/stg28ff68ea-caa6-47bf-be5e-67394ab72f64/working_directory_1.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01846] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_3] Max memory used: 239MiB\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_3] completed success\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_3] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_dict_25\n\u001B[1;30mINFO\u001B[0m [step get_dict_25] start\n\u001B[1;30mINFO\u001B[0m [job get_dict_25] /tmp/zrs68r3d$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/jj2_r82j/stg13ba3696-061f-46f7-b012-bb7ab8a40796/kpts.pickle \\\n --arg_structure=/tmp/jj2_r82j/stg5ca9e71a-98fe-44df-8ca5-550908525a4e/s_2.pickle \\\n --arg_pseudopotentials=/tmp/jj2_r82j/stg9e1fe3f0-1192-4cde-9772-6be8dfa314fe/pseudopotentials.pickle \\\n --arg_calculation=/tmp/jj2_r82j/stgde664722-b13c-4ae9-9562-aec2063d548a/calculation_1.pickle \\\n --arg_smearing=/tmp/jj2_r82j/stg799e1f26-f08e-40bf-a7a9-c58e907674c8/smearing.pickle\n\u001B[1;30mINFO\u001B[0m [job get_dict_25] completed success\n\u001B[1;30mINFO\u001B[0m [step get_dict_25] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_dict_23\n\u001B[1;30mINFO\u001B[0m [step get_dict_23] start\n\u001B[1;30mINFO\u001B[0m [job get_dict_23] /tmp/7qgifyu_$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/ojouuynn/stg7b99ab8b-9235-4007-b6b7-dd4170bd1760/kpts.pickle \\\n --arg_structure=/tmp/ojouuynn/stg8bfd87ff-e83d-4918-9a0d-71b958c46154/s_1.pickle \\\n --arg_pseudopotentials=/tmp/ojouuynn/stg0bcc3451-1700-46e0-9a5e-fca9968c8521/pseudopotentials.pickle \\\n --arg_calculation=/tmp/ojouuynn/stgb9d8de26-7d88-4119-ad13-8374ff498f72/calculation_1.pickle \\\n --arg_smearing=/tmp/ojouuynn/stg01fe91f8-b588-4adc-b207-9d9d4e0956e3/smearing.pickle\n\u001B[1;30mINFO\u001B[0m [job get_dict_23] completed success\n\u001B[1;30mINFO\u001B[0m [step get_dict_23] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_dict_27\n\u001B[1;30mINFO\u001B[0m [step get_dict_27] start\n\u001B[1;30mINFO\u001B[0m [job get_dict_27] /tmp/v83aah99$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/_wg2md43/stgee67f0e5-87a9-4768-82ed-cb6c6cf74b6d/kpts.pickle \\\n --arg_structure=/tmp/_wg2md43/stgb39aa4d9-0909-4ffb-bfe8-c448a3d77669/s_3.pickle \\\n --arg_pseudopotentials=/tmp/_wg2md43/stgc7b618ee-6e90-4b93-a079-785fab087237/pseudopotentials.pickle \\\n --arg_calculation=/tmp/_wg2md43/stg738933fd-9a31-4c0e-ac99-984d1002fd0a/calculation_1.pickle \\\n --arg_smearing=/tmp/_wg2md43/stg174b61ec-1446-4c63-a433-49a940e48f39/smearing.pickle\n\u001B[1;30mINFO\u001B[0m [job get_dict_27] completed success\n\u001B[1;30mINFO\u001B[0m [step get_dict_27] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step calculate_qe_4\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_4] start\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_4] /tmp/abio0491$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/pltanpun/stg312b10e4-f5f2-4770-b776-12c659453bef/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/pltanpun/stgceefbf38-ae57-4369-872d-4ea0895a6976/result.pickle \\\n --arg_working_directory=/tmp/pltanpun/stg6abc0b8b-ed0d-4692-b100-b510fe7de457/working_directory_2.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01928] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_4] Max memory used: 244MiB\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_4] completed success\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_4] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_dict_29\n\u001B[1;30mINFO\u001B[0m [step get_dict_29] start\n\u001B[1;30mINFO\u001B[0m [job get_dict_29] /tmp/li__ygke$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/20wxb6mt/stg4f9c6f04-e0da-4f1d-9b91-17867111fcec/kpts.pickle \\\n --arg_structure=/tmp/20wxb6mt/stg27aeb042-5dba-406d-ad3f-c1a15414d462/s_4.pickle \\\n --arg_pseudopotentials=/tmp/20wxb6mt/stg00a521b7-0d6f-44fa-849f-611e82b816d2/pseudopotentials.pickle \\\n --arg_calculation=/tmp/20wxb6mt/stg51a4e465-9a6d-465e-a329-ec65c9631dca/calculation_1.pickle \\\n --arg_smearing=/tmp/20wxb6mt/stg72f4b40d-8647-441e-94ce-b4d60eb84837/smearing.pickle\n\u001B[1;30mINFO\u001B[0m [job get_dict_29] completed success\n\u001B[1;30mINFO\u001B[0m [step get_dict_29] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step calculate_qe_7\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_7] start\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_7] /tmp/26ontyw1$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3e7bw8e2/stg86169764-ce4e-40f6-9b63-ef5e11208258/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3e7bw8e2/stge349561c-3f24-48da-94fa-2d16e4df4779/result.pickle \\\n --arg_working_directory=/tmp/3e7bw8e2/stg30622d6d-1c64-4440-9d76-7a70e081ff5e/working_directory_5.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02006] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_7] Max memory used: 243MiB\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_7] completed success\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_7] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step calculate_qe_5\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_5] start\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_5] /tmp/nwlrltkk$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/q_jjx0ja/stgb8b85eae-791f-4e34-9696-ceb277f72155/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/q_jjx0ja/stg2efa8335-1138-47c3-b9e9-d8ea2d8754c0/result.pickle \\\n --arg_working_directory=/tmp/q_jjx0ja/stg4f726c25-7adb-4ce0-970e-bd9ddc5191cc/working_directory_3.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02086] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_5] Max memory used: 246MiB\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_5] completed success\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_5] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step calculate_qe_6\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_6] start\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_6] /tmp/9yp55i6u$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/1c_enl4x/stg2aeb7a80-3364-48a0-810d-955e246aa33b/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/1c_enl4x/stg19bec723-020c-4d0e-bca0-8124f6962c7e/result.pickle \\\n --arg_working_directory=/tmp/1c_enl4x/stgc36f93b3-f6f6-4fec-8eb2-0de1ca22f3cd/working_directory_4.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02164] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_6] Max memory used: 243MiB\n\u001B[1;30mINFO\u001B[0m [job calculate_qe_6] completed success\n\u001B[1;30mINFO\u001B[0m [step calculate_qe_6] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_list_31\n\u001B[1;30mINFO\u001B[0m [step get_list_31] start\n\u001B[1;30mINFO\u001B[0m [job get_list_31] /tmp/xgme48c0$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/u2adengu/stg036d73c4-50fd-4b84-9054-ef00e66e0ea8/energy.pickle \\\n --arg_4=/tmp/u2adengu/stg8736d42d-b110-43b3-8e1d-da22b9883ff5/energy.pickle \\\n --arg_1=/tmp/u2adengu/stg8cac7eb7-6132-4085-865d-673027e26baa/energy.pickle \\\n --arg_3=/tmp/u2adengu/stg55b370b4-c852-4493-bb7b-4acaa0588aa8/energy.pickle \\\n --arg_0=/tmp/u2adengu/stg5d319d67-b4fc-4c4d-886a-529b9bfd2458/energy.pickle\n\u001B[1;30mINFO\u001B[0m [job get_list_31] completed success\n\u001B[1;30mINFO\u001B[0m [step get_list_31] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step get_list_30\n\u001B[1;30mINFO\u001B[0m [step get_list_30] start\n\u001B[1;30mINFO\u001B[0m [job get_list_30] /tmp/1xbu0rii$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/0ads3xzd/stg24fc8770-2739-45d1-b535-89924a6234ed/volume.pickle \\\n --arg_4=/tmp/0ads3xzd/stgfb79ea5f-6109-4f2e-a1d6-0817648a5350/volume.pickle \\\n --arg_1=/tmp/0ads3xzd/stg0b1e9637-dba2-4e42-9dbe-c605e7469714/volume.pickle \\\n --arg_3=/tmp/0ads3xzd/stgdf4991ae-4266-4ab6-82be-8f87d46fa7f6/volume.pickle \\\n --arg_0=/tmp/0ads3xzd/stg623a06b5-d2a0-44fd-9473-833452ad5762/volume.pickle\n\u001B[1;30mINFO\u001B[0m [job get_list_30] completed success\n\u001B[1;30mINFO\u001B[0m [step get_list_30] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] starting step plot_energy_volume_curve_8\n\u001B[1;30mINFO\u001B[0m [step plot_energy_volume_curve_8] start\n\u001B[1;30mINFO\u001B[0m [job plot_energy_volume_curve_8] /tmp/vcffw_nr$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/6s32h5zf/stg79d5980d-139d-4481-9277-3f58bb27d49f/workflow.py \\\n --function=workflow.plot_energy_volume_curve \\\n --arg_volume_lst=/tmp/6s32h5zf/stg67eb8ed0-f498-4181-a5a5-c7e74ecd3276/result.pickle \\\n --arg_energy_lst=/tmp/6s32h5zf/stg3ebdb224-bd9a-4ae7-ad4e-03489dcfb661/result.pickle\n\u001B[1;30mINFO\u001B[0m [job plot_energy_volume_curve_8] Max memory used: 110MiB\n\u001B[1;30mINFO\u001B[0m [job plot_energy_volume_curve_8] completed success\n\u001B[1;30mINFO\u001B[0m [step plot_energy_volume_curve_8] completed success\n\u001B[1;30mINFO\u001B[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/quantum_espresso/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$dbc1aaddc8b7343d6d33b34edcf608b8f8801918\",\n \"size\": 4,\n \"path\": \"/home/jovyan/example_workflows/quantum_espresso/result.pickle\"\n }\n}\u001B[1;30mINFO\u001B[0m Final process status is success\n" + } + ], + "execution_count": 6 + }, + { + "id": "2942dbba-ea0a-4d20-be5c-ed9992d09ff8", + "cell_type": "code", + "source": "with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))", + "metadata": { + "trusted": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "None\n" + } + ], + "execution_count": 7 + }, + { + "id": "60e909ee-d0d0-4bd1-81c8-dd5274ae5834", + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + } + ] +} diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py index e69de29..ee87a7c 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py @@ -0,0 +1,240 @@ +import json +import pickle +from yaml import CDumper as Dumper, dump + + +from python_workflow_definition.purepython import ( + group_edges, + resort_total_lst, +) +from python_workflow_definition.shared import ( + convert_nodes_list_to_dict, + remove_result, + EDGES_LABEL, + NODES_LABEL, + TARGET_LABEL, + TARGET_PORT_LABEL, + SOURCE_LABEL, + SOURCE_PORT_LABEL, +) + + +def _get_function_argument(argument: str, position: int = 3) -> dict: + return { + argument + + "_file": { + "type": "File", + "inputBinding": { + "prefix": "--arg_" + argument + "=", + "separate": False, + "position": position, + }, + }, + } + + +def _get_function_template(function_name: str) -> dict: + return { + "function": { + "default": function_name, + "inputBinding": {"position": 3, "prefix": "--function=", "separate": False}, + "type": "string", + }, + } + + +def _get_output_name(output_name: str) -> dict: + return { + output_name + + "_file": {"type": "File", "outputBinding": {"glob": output_name + ".pickle"}} + } + + +def _get_function(workflow): + function_nodes_dict = { + n["id"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "function" + } + funct_dict = {} + for funct_id in function_nodes_dict.keys(): + target_ports = list( + set( + [ + e[TARGET_PORT_LABEL] + for e in workflow[EDGES_LABEL] + if e["target"] == funct_id + ] + ) + ) + source_ports = list( + set( + [ + e[SOURCE_PORT_LABEL] + for e in workflow[EDGES_LABEL] + if e["source"] == funct_id + ] + ) + ) + funct_dict[funct_id] = { + "targetPorts": target_ports, + "sourcePorts": source_ports, + } + return function_nodes_dict, funct_dict + + +def _write_function_cwl(workflow): + function_nodes_dict, funct_dict = _get_function(workflow) + + for i in function_nodes_dict.keys(): + template = { + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "baseCommand": "python", + "inputs": { + "wrapper": { + "type": "string", + "inputBinding": {"position": 1, "prefix": "-m"}, + "default": "python_workflow_definition.cwl", + }, + "workflowfile": { + "type": "File", + "inputBinding": { + "position": 2, + "prefix": "--workflowfile=", + "separate": False, + }, + "default": {"class": "File", "location": "workflow.py"}, + }, + }, + "outputs": {}, + } + file_name = function_nodes_dict[i].split(".")[-1] + "_" + str(i) + ".cwl" + if function_nodes_dict[i].split(".")[0] != "python_workflow_definition": + template["inputs"]["workflowfile"]["default"]["location"] = ( + function_nodes_dict[i].split(".")[0] + ".py" + ) + else: + del template["inputs"]["workflowfile"] + template["inputs"].update( + _get_function_template(function_name=function_nodes_dict[i]) + ) + for j, arg in enumerate(funct_dict[i]["targetPorts"]): + template["inputs"].update( + _get_function_argument(argument=arg, position=4 + j) + ) + for out in funct_dict[i]["sourcePorts"]: + if out is None: + template["outputs"].update(_get_output_name(output_name="result")) + else: + template["outputs"].update(_get_output_name(output_name=out)) + with open(file_name, "w") as f: + dump(template, f, Dumper=Dumper) + + +def _write_workflow_config(workflow): + input_dict = { + n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" + } + with open("workflow.yml", "w") as f: + dump( + { + k + "_file": {"class": "File", "path": k + ".pickle"} + for k in input_dict.keys() + }, + f, + Dumper=Dumper, + ) + for k, v in input_dict.items(): + with open(k + ".pickle", "wb") as f: + pickle.dump(v, f) + + +def _write_workflow(workflow): + workflow_template = { + "cwlVersion": "v1.2", + "class": "Workflow", + "inputs": {}, + "steps": {}, + "outputs": {}, + } + input_dict = { + n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" + } + function_nodes_dict, funct_dict = _get_function(workflow) + result_id = [n["id"] for n in workflow[NODES_LABEL] if n["type"] == "output"][0] + last_compute_id = [ + e[SOURCE_LABEL] for e in workflow[EDGES_LABEL] if e[TARGET_LABEL] == result_id + ][0] + workflow_template["inputs"].update({k + "_file": "File" for k in input_dict.keys()}) + if funct_dict[last_compute_id]["sourcePorts"] == [None]: + workflow_template["outputs"] = { + "result_file": { + "type": "File", + "outputSource": function_nodes_dict[last_compute_id].split(".")[-1] + + "_" + + str(last_compute_id) + + "/result_file", + }, + } + else: + raise ValueError() + + content = remove_result(workflow_dict=workflow) + edges_new_lst = content[EDGES_LABEL] + total_lst = group_edges(edges_new_lst) + nodes_new_dict = { + int(k): v + for k, v in convert_nodes_list_to_dict(nodes_list=content[NODES_LABEL]).items() + } + total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict) + step_name_lst = { + t[0]: function_nodes_dict[t[0]].split(".")[-1] for t in total_new_lst + } + input_id_dict = { + n["id"]: n["name"] for n in workflow[NODES_LABEL] if n["type"] == "input" + } + for t in total_new_lst: + ind = t[0] + node_script = step_name_lst[ind] + "_" + str(ind) + ".cwl" + output = [ + o + "_file" if o is not None else "result_file" + for o in funct_dict[ind]["sourcePorts"] + ] + in_dict = {} + for k, v in t[1].items(): + if v[SOURCE_LABEL] in input_id_dict: + in_dict[k + "_file"] = input_id_dict[v[SOURCE_LABEL]] + "_file" + else: + if v["sourcePort"] is None: + in_dict[k + "_file"] = ( + step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + + "/result_file" + ) + else: + in_dict[k + "_file"] = ( + step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + + "/" + + v[SOURCE_PORT_LABEL] + + "_file" + ) + workflow_template["steps"].update( + { + step_name_lst[ind] + + "_" + + str(ind): {"run": node_script, "in": in_dict, "out": output} + } + ) + with open("workflow.cwl", "w") as f: + dump(workflow_template, f, Dumper=Dumper) + + +def load_workflow_json(file_name: str): + with open(file_name, "r") as f: + workflow = json.load(f) + + _write_function_cwl(workflow=workflow) + _write_workflow_config(workflow=workflow) + _write_workflow(workflow=workflow) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py deleted file mode 100644 index ee87a7c..0000000 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ /dev/null @@ -1,240 +0,0 @@ -import json -import pickle -from yaml import CDumper as Dumper, dump - - -from python_workflow_definition.purepython import ( - group_edges, - resort_total_lst, -) -from python_workflow_definition.shared import ( - convert_nodes_list_to_dict, - remove_result, - EDGES_LABEL, - NODES_LABEL, - TARGET_LABEL, - TARGET_PORT_LABEL, - SOURCE_LABEL, - SOURCE_PORT_LABEL, -) - - -def _get_function_argument(argument: str, position: int = 3) -> dict: - return { - argument - + "_file": { - "type": "File", - "inputBinding": { - "prefix": "--arg_" + argument + "=", - "separate": False, - "position": position, - }, - }, - } - - -def _get_function_template(function_name: str) -> dict: - return { - "function": { - "default": function_name, - "inputBinding": {"position": 3, "prefix": "--function=", "separate": False}, - "type": "string", - }, - } - - -def _get_output_name(output_name: str) -> dict: - return { - output_name - + "_file": {"type": "File", "outputBinding": {"glob": output_name + ".pickle"}} - } - - -def _get_function(workflow): - function_nodes_dict = { - n["id"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "function" - } - funct_dict = {} - for funct_id in function_nodes_dict.keys(): - target_ports = list( - set( - [ - e[TARGET_PORT_LABEL] - for e in workflow[EDGES_LABEL] - if e["target"] == funct_id - ] - ) - ) - source_ports = list( - set( - [ - e[SOURCE_PORT_LABEL] - for e in workflow[EDGES_LABEL] - if e["source"] == funct_id - ] - ) - ) - funct_dict[funct_id] = { - "targetPorts": target_ports, - "sourcePorts": source_ports, - } - return function_nodes_dict, funct_dict - - -def _write_function_cwl(workflow): - function_nodes_dict, funct_dict = _get_function(workflow) - - for i in function_nodes_dict.keys(): - template = { - "cwlVersion": "v1.2", - "class": "CommandLineTool", - "baseCommand": "python", - "inputs": { - "wrapper": { - "type": "string", - "inputBinding": {"position": 1, "prefix": "-m"}, - "default": "python_workflow_definition.cwl", - }, - "workflowfile": { - "type": "File", - "inputBinding": { - "position": 2, - "prefix": "--workflowfile=", - "separate": False, - }, - "default": {"class": "File", "location": "workflow.py"}, - }, - }, - "outputs": {}, - } - file_name = function_nodes_dict[i].split(".")[-1] + "_" + str(i) + ".cwl" - if function_nodes_dict[i].split(".")[0] != "python_workflow_definition": - template["inputs"]["workflowfile"]["default"]["location"] = ( - function_nodes_dict[i].split(".")[0] + ".py" - ) - else: - del template["inputs"]["workflowfile"] - template["inputs"].update( - _get_function_template(function_name=function_nodes_dict[i]) - ) - for j, arg in enumerate(funct_dict[i]["targetPorts"]): - template["inputs"].update( - _get_function_argument(argument=arg, position=4 + j) - ) - for out in funct_dict[i]["sourcePorts"]: - if out is None: - template["outputs"].update(_get_output_name(output_name="result")) - else: - template["outputs"].update(_get_output_name(output_name=out)) - with open(file_name, "w") as f: - dump(template, f, Dumper=Dumper) - - -def _write_workflow_config(workflow): - input_dict = { - n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" - } - with open("workflow.yml", "w") as f: - dump( - { - k + "_file": {"class": "File", "path": k + ".pickle"} - for k in input_dict.keys() - }, - f, - Dumper=Dumper, - ) - for k, v in input_dict.items(): - with open(k + ".pickle", "wb") as f: - pickle.dump(v, f) - - -def _write_workflow(workflow): - workflow_template = { - "cwlVersion": "v1.2", - "class": "Workflow", - "inputs": {}, - "steps": {}, - "outputs": {}, - } - input_dict = { - n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" - } - function_nodes_dict, funct_dict = _get_function(workflow) - result_id = [n["id"] for n in workflow[NODES_LABEL] if n["type"] == "output"][0] - last_compute_id = [ - e[SOURCE_LABEL] for e in workflow[EDGES_LABEL] if e[TARGET_LABEL] == result_id - ][0] - workflow_template["inputs"].update({k + "_file": "File" for k in input_dict.keys()}) - if funct_dict[last_compute_id]["sourcePorts"] == [None]: - workflow_template["outputs"] = { - "result_file": { - "type": "File", - "outputSource": function_nodes_dict[last_compute_id].split(".")[-1] - + "_" - + str(last_compute_id) - + "/result_file", - }, - } - else: - raise ValueError() - - content = remove_result(workflow_dict=workflow) - edges_new_lst = content[EDGES_LABEL] - total_lst = group_edges(edges_new_lst) - nodes_new_dict = { - int(k): v - for k, v in convert_nodes_list_to_dict(nodes_list=content[NODES_LABEL]).items() - } - total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict) - step_name_lst = { - t[0]: function_nodes_dict[t[0]].split(".")[-1] for t in total_new_lst - } - input_id_dict = { - n["id"]: n["name"] for n in workflow[NODES_LABEL] if n["type"] == "input" - } - for t in total_new_lst: - ind = t[0] - node_script = step_name_lst[ind] + "_" + str(ind) + ".cwl" - output = [ - o + "_file" if o is not None else "result_file" - for o in funct_dict[ind]["sourcePorts"] - ] - in_dict = {} - for k, v in t[1].items(): - if v[SOURCE_LABEL] in input_id_dict: - in_dict[k + "_file"] = input_id_dict[v[SOURCE_LABEL]] + "_file" - else: - if v["sourcePort"] is None: - in_dict[k + "_file"] = ( - step_name_lst[v[SOURCE_LABEL]] - + "_" - + str(v[SOURCE_LABEL]) - + "/result_file" - ) - else: - in_dict[k + "_file"] = ( - step_name_lst[v[SOURCE_LABEL]] - + "_" - + str(v[SOURCE_LABEL]) - + "/" - + v[SOURCE_PORT_LABEL] - + "_file" - ) - workflow_template["steps"].update( - { - step_name_lst[ind] - + "_" - + str(ind): {"run": node_script, "in": in_dict, "out": output} - } - ) - with open("workflow.cwl", "w") as f: - dump(workflow_template, f, Dumper=Dumper) - - -def load_workflow_json(file_name: str): - with open(file_name, "r") as f: - workflow = json.load(f) - - _write_function_cwl(workflow=workflow) - _write_workflow_config(workflow=workflow) - _write_workflow(workflow=workflow)