Merge pull request #1470 from pyiron/h5io_update

Update the h5io interface

.ci_support/environment-docs.yml

@@ -7,7 +7,7 @@ dependencies:
 - myst-parser
 - cloudpickle =3.0.0
 - gitpython =3.1.43
-- h5io_browser =0.0.12
+- h5io_browser =0.0.14
 - h5py =3.11.0
 - jinja2 =3.1.4
 - monty =2024.5.24

.ci_support/environment-old.yml

@@ -3,7 +3,7 @@ channels:
 dependencies:
 - cloudpickle =2.0.0
 - gitpython =3.1.23
-- h5io_browser =0.0.6
+- h5io_browser =0.0.14
 - h5py =3.6.0
 - jinja2 =2.11.3
 - monty =2021.3.3

.ci_support/environment.yml

@@ -7,7 +7,7 @@ dependencies:
 - conda_subprocess =0.0.1
 - cloudpickle =3.0.0
 - gitpython =3.1.43
-- h5io_browser =0.0.12
+- h5io_browser =0.0.14
 - h5py =3.11.0
 - jinja2 =3.1.4
 - monty =2024.5.24

pyiron_base/storage/hdfio.py

@@ -6,7 +6,8 @@
 """
 
 import numbers
-from h5io_browser import Pointer
+import h5py
+from h5io_browser import Pointer, read_nested_dict_from_hdf
 from h5io_browser.base import (
     _open_hdf,
     _is_ragged_in_1st_dim_only,
@@ -22,11 +23,6 @@
 import sys
 from typing import Union, Optional, Any, Tuple
 
-from pyiron_base.storage.helper_functions import (
-    get_h5_path,
-    list_groups_and_nodes,
-    read_dict_from_hdf,
-)
 from pyiron_base.interfaces.has_groups import HasGroups
 from pyiron_base.state import state
 from pyiron_base.jobs.job.util import _get_safe_job_name
@@ -54,6 +50,29 @@ def _extract_module_class_name(type_field: str) -> Tuple[str, str]:
     return fully_qualified_path.rsplit(".", maxsplit=1)
 
 
+def _list_groups_and_nodes(hdf, h5_path):
+    """
+    Get the list of groups and list of nodes from an open HDF5 file
+    Args:
+        hdf (h5py.File): file handle of an open HDF5 file
+        h5_path (str): path inside the HDF5 file
+    Returns:
+        list, list: list of groups and list of nodes
+    """
+    groups = set()
+    nodes = set()
+    try:
+        h = hdf[h5_path]
+        for k in h.keys():
+            if isinstance(h[k], h5py.Group):
+                groups.add(k)
+            else:
+                nodes.add(k)
+    except KeyError:
+        pass
+    return list(groups), list(nodes)
+
+
 def _import_class(module_path, class_name):
     """
     Import given class from fully qualified name and return class object.
@@ -520,7 +539,7 @@ def _list_all(self):
         """
         if self.file_exists:
             with _open_hdf(self.file_name) as hdf:
-                groups, nodes = list_groups_and_nodes(hdf=hdf, h5_path=self.h5_path)
+                groups, nodes = _list_groups_and_nodes(hdf=hdf, h5_path=self.h5_path)
             iopy_nodes = self._filter_io_objects(set(groups))
             return {
                 "groups": sorted(list(set(groups) - iopy_nodes)),
@@ -748,7 +767,7 @@ def read_dict_from_hdf(self, group_paths=[], recursive=False):
         Returns:
             dict:     The loaded data. Can be of any type supported by ``write_hdf5``.
         """
-        return read_dict_from_hdf(
+        return read_nested_dict_from_hdf(
             file_name=self.file_name,
             h5_path=self.h5_path,
             group_paths=group_paths,
@@ -777,7 +796,7 @@ def _get_h5_path(self, name):
         Returns:
             str: combined path
         """
-        return get_h5_path(h5_path=self.h5_path, name=name)
+        return posixpath.join(self.h5_path, name)
 
     def _get_h5io_type(self, name):
         """

pyproject.toml

@@ -26,7 +26,7 @@ classifiers = [
 dependencies = [
     "cloudpickle==3.0.0",
     "gitpython==3.1.43",
-    "h5io_browser==0.0.12",
+    "h5io_browser==0.0.14",
     "h5py==3.11.0",
     "jinja2==3.1.4",
     "numpy==1.26.4",

tests/storage/test_helper_functions.py

@@ -2,10 +2,8 @@
 import posixpath
 import h5py
 from unittest import TestCase
-from pyiron_base.storage.helper_functions import (
-    list_groups_and_nodes,
-    read_dict_from_hdf,
-)
+from pyiron_base.storage.hdfio import _list_groups_and_nodes
+from h5io_browser import read_nested_dict_from_hdf
 from h5io_browser.base import write_dict_to_hdf, _read_hdf, _write_hdf
 
 
@@ -44,11 +42,11 @@ def test_read_hierarchical(self):
     def test_read_dict_hierarchical(self):
         self.assertEqual(
             {"key_b": 3},
-            read_dict_from_hdf(file_name=self.file_name, h5_path=self.h5_path),
+            read_nested_dict_from_hdf(file_name=self.file_name, h5_path=self.h5_path),
         )
         self.assertEqual(
             {"key_a": {"idx_0": 1, "idx_1": 2}, "key_b": 3, "key_c": {"key_d": 4}},
-            read_dict_from_hdf(
+            read_nested_dict_from_hdf(
                 file_name=self.file_name,
                 h5_path=self.h5_path,
                 group_paths=["key_a", "key_c"],
@@ -60,7 +58,7 @@ def test_read_dict_hierarchical(self):
                 "key_b": 3,
                 "key_c": {"key_d": 4, "key_e": {"key_f": 5}},
             },
-            read_dict_from_hdf(
+            read_nested_dict_from_hdf(
                 file_name=self.file_name,
                 h5_path=self.h5_path,
                 group_paths=["key_a", "key_c", "key_c/key_e"],
@@ -72,7 +70,7 @@ def test_read_dict_hierarchical(self):
                 "key_b": 3,
                 "key_c": {"key_d": 4, "key_e": {"key_f": 5}},
             },
-            read_dict_from_hdf(
+            read_nested_dict_from_hdf(
                 file_name=self.file_name,
                 h5_path=self.h5_path,
                 recursive=True,
@@ -106,7 +104,7 @@ def test_hdf5_structure(self):
 
     def test_list_groups(self):
         with h5py.File(self.file_name, "r") as f:
-            groups, nodes = list_groups_and_nodes(hdf=f, h5_path="data_hierarchical")
+            groups, nodes = _list_groups_and_nodes(hdf=f, h5_path="data_hierarchical")
         self.assertEqual(list(sorted(groups)), ["key_a", "key_c"])
         self.assertEqual(nodes, ["key_b"])
 
@@ -134,7 +132,7 @@ def test_read_hierarchical(self):
     def test_read_dict_hierarchical(self):
         self.assertEqual(
             self.data_hierarchical,
-            read_dict_from_hdf(file_name=self.file_name, h5_path=self.h5_path),
+            read_nested_dict_from_hdf(file_name=self.file_name, h5_path=self.h5_path),
         )
 
     def test_hdf5_structure(self):
@@ -150,6 +148,6 @@ def test_hdf5_structure(self):
 
     def test_list_groups(self):
         with h5py.File(self.file_name, "r") as f:
-            groups, nodes = list_groups_and_nodes(hdf=f, h5_path="data_hierarchical")
+            groups, nodes = _list_groups_and_nodes(hdf=f, h5_path="data_hierarchical")
         self.assertEqual(groups, [])
         self.assertEqual(list(sorted(nodes)), ["a", "b", "c"])