Skip to content

Reduce JSON format #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 30, 2025
Merged

Reduce JSON format #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d433f53
Reduce JSON format
jan-janssen Mar 30, 2025
915108f
update readme
jan-janssen Mar 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,15 @@ JSON file:
"5": 2
},
"edges": [
{"target": 0, "targetHandle": "x", "source": 1, "sourceHandle": "x"},
{"target": 1, "targetHandle": "x", "source": 4, "sourceHandle": null},
{"target": 1, "targetHandle": "y", "source": 5, "sourceHandle": null},
{"target": 0, "targetHandle": "y", "source": 2, "sourceHandle": "y"},
{"target": 2, "targetHandle": "x", "source": 4, "sourceHandle": null},
{"target": 2, "targetHandle": "y", "source": 5, "sourceHandle": null},
{"target": 0, "targetHandle": "z", "source": 3, "sourceHandle": "z"},
{"target": 3, "targetHandle": "x", "source": 4, "sourceHandle": null},
{"target": 3, "targetHandle": "y", "source": 5, "sourceHandle": null}
{"tn": 0, "th": "x", "sn": 1, "sh": "x"},
{"tn": 1, "th": "x", "sn": 4, "sh": null},
{"tn": 1, "th": "y", "sn": 5, "sh": null},
{"tn": 0, "th": "y", "sn": 2, "sh": "y"},
{"tn": 2, "th": "x", "sn": 4, "sh": null},
{"tn": 2, "th": "y", "sn": 5, "sh": null},
{"tn": 0, "th": "z", "sn": 3, "sh": "z"},
{"tn": 3, "th": "x", "sn": 4, "sh": null},
{"tn": 3, "th": "y", "sn": 5, "sh": null}
]
}
```
Expand Down
32 changes: 16 additions & 16 deletions python_workflow_definition/src/python_workflow_definition/aiida.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,34 +35,34 @@ def load_workflow_json(file_name):
task_name_mapping[id] = wg.tasks[-1].name
# add links
for link in data["edges"]:
if link["sourceHandle"] is None:
link["sourceHandle"] = "result"
if link["sh"] is None:
link["sh"] = "result"
try:
from_task = wg.tasks[task_name_mapping[str(link["source"])]]
from_task = wg.tasks[task_name_mapping[str(link["sn"])]]
# because we are not define the outputs explicitly during the pythonjob creation
# we add it here, and assume the output exit
if link["sourceHandle"] not in from_task.outputs:
# if str(link["sourceHandle"]) not in from_task.outputs:
if link["sh"] not in from_task.outputs:
# if str(link["sh"]) not in from_task.outputs:
from_socket = from_task.add_output(
"workgraph.any",
name=link["sourceHandle"],
# name=str(link["sourceHandle"]),
name=link["sh"],
# name=str(link["sh"]),
metadata={"is_function_output": True},
)
else:
from_socket = from_task.outputs[link["sourceHandle"]]
to_task = wg.tasks[task_name_mapping[str(link["target"])]]
from_socket = from_task.outputs[link["sh"]]
to_task = wg.tasks[task_name_mapping[str(link["tn"])]]
# if the input is not exit, it means we pass the data into to the kwargs
# in this case, we add the input socket
if link["targetHandle"] not in to_task.inputs:
if link["th"] not in to_task.inputs:
#
to_socket = to_task.add_input(
"workgraph.any",
name=link["targetHandle"],
name=link["th"],
metadata={"is_function_input": True},
)
else:
to_socket = to_task.inputs[link["targetHandle"]]
to_socket = to_task.inputs[link["th"]]
wg.add_link(from_socket, to_socket)
except Exception as e:
traceback.print_exc()
Expand Down Expand Up @@ -112,10 +112,10 @@ def write_workflow_json(wg, file_name):
# if the from socket is the default result, we set it to None
if link_data["from_socket"] == "result":
link_data["from_socket"] = None
link_data["target"] = node_name_mapping[link_data.pop("to_node")]
link_data["targetHandle"] = link_data.pop("to_socket")
link_data["source"] = node_name_mapping[link_data.pop("from_node")]
link_data["sourceHandle"] = link_data.pop("from_socket")
link_data["tn"] = node_name_mapping[link_data.pop("to_node")]
link_data["th"] = link_data.pop("to_socket")
link_data["sn"] = node_name_mapping[link_data.pop("from_node")]
link_data["sh"] = link_data.pop("from_socket")
data["edges"].append(link_data)

with open(file_name, "w") as f:
Expand Down
2 changes: 1 addition & 1 deletion python_workflow_definition/src/python_workflow_definition/executorlib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def get_item(obj, key):


def _get_value(result_dict, nodes_new_dict, link_dict, exe):
source, source_handle = link_dict["source"], link_dict["sourceHandle"]
source, source_handle = link_dict["sn"], link_dict["sh"]
if source in result_dict.keys():
result = result_dict[source]
elif source in nodes_new_dict.keys():
Expand Down
34 changes: 17 additions & 17 deletions python_workflow_definition/src/python_workflow_definition/jobflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ def _get_nodes_dict(function_dict):

def _get_edge_from_dict(target, key, value_dict, nodes_mapping_dict):
if len(value_dict['attributes']) == 1:
return {'target': target, 'targetHandle': key, "source": nodes_mapping_dict[value_dict['uuid']], 'sourceHandle': value_dict['attributes'][0][1]}
return {'tn': target, 'th': key, "sn": nodes_mapping_dict[value_dict['uuid']], 'sh': value_dict['attributes'][0][1]}
else:
return {'target': target, 'targetHandle': key, "source": nodes_mapping_dict[value_dict['uuid']], 'sourceHandle': None}
return {'tn': target, 'th': key, "sn": nodes_mapping_dict[value_dict['uuid']], 'sh': None}


def _get_edges_and_extend_nodes(flow_dict, nodes_mapping_dict, nodes_dict):
Expand Down Expand Up @@ -59,8 +59,8 @@ def _get_edges_and_extend_nodes(flow_dict, nodes_mapping_dict, nodes_dict):
nodes_dict[node_index] = vt
else:
node_index = {str(tv): tk for tk, tv in nodes_dict.items()}[str(vt)]
edges_lst.append({'target': node_dict_index, 'targetHandle': kt, "source": node_index, 'sourceHandle': None})
edges_lst.append({'target': nodes_mapping_dict[job["uuid"]], 'targetHandle': k, "source": node_dict_index, 'sourceHandle': None})
edges_lst.append({'tn': node_dict_index, 'th': kt, "sn": node_index, 'sh': None})
edges_lst.append({'tn': nodes_mapping_dict[job["uuid"]], 'th': k, "sn": node_dict_index, 'sh': None})
elif isinstance(v, list) and any([isinstance(el, dict) and '@module' in el and '@class' in el and '@version' in el for el in v]):
node_list_index = len(nodes_dict)
nodes_dict[node_list_index] = get_list
Expand All @@ -78,15 +78,15 @@ def _get_edges_and_extend_nodes(flow_dict, nodes_mapping_dict, nodes_dict):
nodes_dict[node_index] = vt
else:
node_index = {str(tv): tk for tk, tv in nodes_dict.items()}[str(vt)]
edges_lst.append({'target': node_list_index, 'targetHandle': kt, "source": node_index, 'sourceHandle': None})
edges_lst.append({'target': nodes_mapping_dict[job["uuid"]], 'targetHandle': k, "source": node_list_index, 'sourceHandle': None})
edges_lst.append({'tn': node_list_index, 'th': kt, "sn": node_index, 'sh': None})
edges_lst.append({'tn': nodes_mapping_dict[job["uuid"]], 'th': k, "sn": node_list_index, 'sh': None})
else:
if v not in nodes_dict.values():
node_index = len(nodes_dict)
nodes_dict[node_index] = v
else:
node_index = {tv: tk for tk, tv in nodes_dict.items()}[v]
edges_lst.append({'target': nodes_mapping_dict[job["uuid"]], 'targetHandle': k, "source": node_index, 'sourceHandle': None})
edges_lst.append({'tn': nodes_mapping_dict[job["uuid"]], 'th': k, "sn": node_index, 'sh': None})
return edges_lst, nodes_dict


Expand All @@ -99,7 +99,7 @@ def _resort_total_lst(total_dict, nodes_dict):
for ind in sorted(total_dict.keys()):
connect = total_dict[ind]
if ind not in ordered_lst:
source_lst = [sd["source"] for sd in connect.values()]
source_lst = [sd["sn"] for sd in connect.values()]
if all([s in ordered_lst or s in nodes_without_dep_lst for s in source_lst]):
ordered_lst.append(ind)
total_new_dict[ind] = connect
Expand All @@ -109,11 +109,11 @@ def _resort_total_lst(total_dict, nodes_dict):
def _group_edges(edges_lst):
total_dict = {}
for ed_major in edges_lst:
target_id = ed_major["target"]
target_id = ed_major["tn"]
tmp_lst = []
if target_id not in total_dict.keys():
for ed in edges_lst:
if target_id == ed["target"]:
if target_id == ed["tn"]:
tmp_lst.append(ed)
total_dict[target_id] = get_kwargs(lst=tmp_lst)
return total_dict
Expand All @@ -139,8 +139,8 @@ def get_attr_helper(obj, source_handle):
else:
fn = job(method=v)
kwargs = {
kw: input_dict[vw['source']] if vw['source'] in input_dict else get_attr_helper(
obj=memory_dict[vw['source']], source_handle=vw['sourceHandle'])
kw: input_dict[vw['sn']] if vw['sn'] in input_dict else get_attr_helper(
obj=memory_dict[vw['sn']], source_handle=vw['sh'])
for kw, vw in total_dict[k].items()
}
memory_dict[k] = fn(**kwargs)
Expand All @@ -160,15 +160,15 @@ def load_workflow_json(file_name):

edges_new_lst = []
for edge in content["edges"]:
if edge['sourceHandle'] is None:
if edge['sh'] is None:
edges_new_lst.append(edge)
else:
edges_new_lst.append(
{
'target': edge['target'],
'targetHandle': edge['targetHandle'],
'source': edge['source'],
'sourceHandle': str(edge['sourceHandle']),
'tn': edge['tn'],
'th': edge['th'],
'sn': edge['sn'],
'sh': str(edge['sh']),
}
)

Expand Down
12 changes: 6 additions & 6 deletions python_workflow_definition/src/python_workflow_definition/purepython.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,30 @@ def resort_total_lst(total_lst, nodes_dict):
while len(total_new_lst) < len(total_lst):
for ind, connect in total_lst:
if ind not in ordered_lst:
source_lst = [sd["source"] for sd in connect.values()]
source_lst = [sd["sn"] for sd in connect.values()]
if all([s in ordered_lst or s in nodes_without_dep_lst for s in source_lst]):
ordered_lst.append(ind)
total_new_lst.append([ind, connect])
return total_new_lst


def group_edges(edges_lst):
edges_sorted_lst = sorted(edges_lst, key=lambda x: x['target'], reverse=True)
edges_sorted_lst = sorted(edges_lst, key=lambda x: x["tn"], reverse=True)
total_lst, tmp_lst = [], []
target_id = edges_sorted_lst[0]['target']
target_id = edges_sorted_lst[0]["tn"]
for ed in edges_sorted_lst:
if target_id == ed["target"]:
if target_id == ed["tn"]:
tmp_lst.append(ed)
else:
total_lst.append((target_id, get_kwargs(lst=tmp_lst)))
target_id = ed["target"]
target_id = ed["tn"]
tmp_lst = [ed]
total_lst.append((target_id, get_kwargs(lst=tmp_lst)))
return total_lst


def _get_value(result_dict, nodes_new_dict, link_dict):
source, source_handle = link_dict["source"], link_dict["sourceHandle"]
source, source_handle = link_dict["sn"], link_dict["sh"]
if source in result_dict.keys():
result = result_dict[source]
elif source in nodes_new_dict.keys():
Expand Down
38 changes: 19 additions & 19 deletions python_workflow_definition/src/python_workflow_definition/pyiron_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,23 @@ def _resort_total_lst(total_lst, nodes_dict):
while len(total_new_lst) < len(total_lst):
for ind, connect in total_lst:
if ind not in ordered_lst:
source_lst = [sd["source"] for sd in connect.values()]
source_lst = [sd["sn"] for sd in connect.values()]
if all([s in ordered_lst or s in nodes_without_dep_lst for s in source_lst]):
ordered_lst.append(ind)
total_new_lst.append([ind, connect])
return total_new_lst


def _group_edges(edges_lst):
edges_sorted_lst = sorted(edges_lst, key=lambda x: x['target'], reverse=True)
edges_sorted_lst = sorted(edges_lst, key=lambda x: x["tn"], reverse=True)
total_lst, tmp_lst = [], []
target_id = edges_sorted_lst[0]['target']
target_id = edges_sorted_lst[0]["tn"]
for ed in edges_sorted_lst:
if target_id == ed["target"]:
if target_id == ed["tn"]:
tmp_lst.append(ed)
else:
total_lst.append((target_id, get_kwargs(lst=tmp_lst)))
target_id = ed["target"]
target_id = ed["tn"]
tmp_lst = [ed]
total_lst.append((target_id, get_kwargs(lst=tmp_lst)))
return total_lst
Expand All @@ -53,8 +53,8 @@ def _get_delayed_object_dict(total_lst, nodes_dict, source_handle_dict, pyiron_p
k: _get_source(
nodes_dict=nodes_dict,
delayed_object_dict=delayed_object_dict,
source=v["source"],
sourceHandle=v["sourceHandle"],
source=v["sn"],
sourceHandle=v["sh"],
)
for k, v in input_dict.items()
}
Expand Down Expand Up @@ -151,24 +151,24 @@ def _get_edges_dict(edges_lst, nodes_dict, connection_dict, lookup_dict):
if isinstance(output, DelayedObject):
if output._list_index is not None:
edges_dict_lst.append({
"target": target,
"targetHandle": target_handle,
"source": connection_dict[output_name],
"sourceHandle": f"s_{output._list_index}", # check for list index
"tn": target,
"th": target_handle,
"sn": connection_dict[output_name],
"sh": f"s_{output._list_index}", # check for list index
})
else:
edges_dict_lst.append({
"target": target,
"targetHandle": target_handle,
"source": connection_dict[output_name],
"sourceHandle": output._output_key, # check for list index
"tn": target,
"th": target_handle,
"sn": connection_dict[output_name],
"sh": output._output_key, # check for list index
})
else:
edges_dict_lst.append({
"target": target,
"targetHandle": target_handle,
"source": connection_dict[output_name],
"sourceHandle": None,
"tn": target,
"th": target_handle,
"sn": connection_dict[output_name],
"sh": None,
})
existing_connection_lst.append(connection_name)
return edges_dict_lst
Expand Down
8 changes: 4 additions & 4 deletions python_workflow_definition/src/python_workflow_definition/shared.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,16 @@ def get_list(**kwargs):


def get_kwargs(lst):
return {t['targetHandle']: {'source': t['source'], 'sourceHandle': t['sourceHandle']} for t in lst}
return {t["th"]: {"sn": t["sn"], "sh": t["sh"]} for t in lst}


def get_source_handles(edges_lst):
source_handle_dict = {}
for ed in edges_lst:
if ed['source'] not in source_handle_dict.keys():
source_handle_dict[ed['source']] = [ed['sourceHandle']]
if ed["sn"] not in source_handle_dict.keys():
source_handle_dict[ed["sn"]] = [ed["sh"]]
else:
source_handle_dict[ed['source']].append(ed['sourceHandle'])
source_handle_dict[ed["sn"]].append(ed["sh"])
return {
k: list(range(len(v))) if len(v) > 1 and all([el is None for el in v]) else v
for k, v in source_handle_dict.items()
Expand Down
Loading
Loading