Functionality enhancements to address lazy loading of chunked data, variable length strings, and other minor bug fixes

.github/workflows/pytest.yml

@@ -5,7 +5,7 @@ name: Python package
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, mock_s3fs ]
   pull_request:
     branches: [ master ]
 
@@ -16,19 +16,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
-        python -m pip install -e .
+        python -m pip install .[testing]  # install in test mode
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |

.gitignore

@@ -1,2 +1,10 @@
 .coverage
+.pyc
 build
+__pycache__/
+*.egg-info
+.idea
+.DS_Store
+test-reports/
+<_io.Bytes*>
+tests/__pycache__/

MANIFEST.in

@@ -1,2 +1,3 @@
 # Include the license file
 include LICENSE.txt
+include README.rst

README.rst

@@ -23,7 +23,7 @@ implemented.
 Dependencies
 ============
 
-pyfive is tested to work with Python 3.8 to 3.13.  It may also work
+pyfive is tested to work with Python 3.10 to 3.13.  It may also work
 with other Python versions.
 
 The only dependencies to run the software besides Python is NumPy.

bnl/opening_speed.py

@@ -0,0 +1,112 @@
+import h5py
+import pyfive
+from pathlib import Path
+import time
+import s3fs
+
+S3_URL = 'https://uor-aces-o.s3-ext.jc.rl.ac.uk/'
+S3_BUCKET = 'bnl'
+
+def test_speed(s3=False):
+
+    mypath = Path(__file__).parent
+    fname1 = 'da193o_25_day__grid_T_198807-198807.nc'
+    vname1 = 'tos'
+    p1 = mypath/fname1
+
+    fname2 = 'ch330a.pc19790301-def-short.nc'
+    vname2 = 'UM_m01s16i202_vn1106'
+    p2 = Path.home()/'Repositories/h5netcdf/h5netcdf/tests/'/fname2
+
+    do_run(p1, fname1, vname1, s3)
+
+    do_run(p2, fname2, vname2, s3)
+
+
+def do_s3(package, fname, vname): 
+
+    fs = s3fs.S3FileSystem(anon=True, client_kwargs={'endpoint_url': S3_URL})
+    uri = S3_BUCKET + '/' + fname
+    with fs.open(uri,'rb') as p:    
+        t_opening, t_var, t_calc, t_tot = do_inner(package, p, vname)
+
+    return t_opening, t_var, t_calc, t_tot
+
+def do_inner(package, p, vname, withdask=False):
+    h0 = time.time()
+    pf1 = package.File(p)
+    h3 = time.time()
+    t_opening = 1000* (h3-h0)
+
+    h5a = time.time()
+    vp = pf1[vname]
+    h5 = time.time()
+    t_var = 1000* (h5-h5a)
+
+    h6a = time.time()
+    sh = sum(vp)
+    h6 = time.time()
+    t_calc = 1000* (h6-h6a)
+
+    t_tot = t_calc+t_var+t_opening
+
+    pf1.close()
+    return t_opening, t_var, t_calc, t_tot
+
+
+
+def do_run(p, fname, vname, s3):
+
+    if s3:
+        import s3fs
+
+
+    # for posix force this to be a comparison from memory
+    # by ensuring file is in disk cache and ignore first access
+    # but we then do an even number of accesses to make sure we are not 
+    # biased by caching. 
+    n = 0
+    datanames = ['h_opening','p_opening','h_var','p_var','h_calc','p_calc','h_tot','p_tot']
+    results = {x:0.0 for x in datanames}
+    while n <2:
+        n+=1
+
+        if s3:
+            h_opening, h_var, h_calc, h_tot = do_s3(h5py, fname, vname)
+            p_opening, p_var, p_calc, p_tot = do_s3(pyfive, fname, vname)
+
+        else:
+            h_opening, h_var, h_calc, h_tot = do_inner(h5py, p, vname)
+            p_opening, p_var, p_calc, p_tot = do_inner(pyfive, p, vname)
+
+        if n>1:
+            for x,r  in zip(datanames,[h_opening,p_opening,h_var,p_var,h_calc,p_calc,h_tot,p_tot]):
+                results[x] += r
+
+    for v in results.values():
+        v = v/(n-1)
+
+
+    print("File Opening Time Comparison ", fname, f' (ms, S3={s3})')
+    print(f"h5py:   {results['h_opening']:9.6f}")
+    print(f"pyfive: {results['p_opening']:9.6f}")
+
+    print(f'Variable instantiation for [{vname}]')
+    print(f"h5py:   {results['h_var']:9.6f}")
+    print(f"pyfive: {results['p_var']:9.6f}")
+
+    print('Access and calculation time for summation')
+    print(f"h5py:   {results['h_calc']:9.6f}")
+    print(f"pyfive: {results['p_calc']:9.6f}")
+
+    print('Total times')
+    print(f"h5py:   {results['h_tot']:9.6f}")
+    print(f"pyfive: {results['p_tot']:9.6f}")
+
+if __name__=="__main__":
+    test_speed()
+    test_speed(s3=True)
+
+
+
+

pyfive/__init__.py

@@ -1,7 +1,13 @@
 """
 pyfive : a pure python HDF5 file reader.
+This is the public API exposed by pyfive,
+which is a small subset of the H5PY API.
 """
 
-from .high_level import File
+from pyfive.high_level import File, Group, Dataset
+from pyfive.h5t import check_enum_dtype, check_string_dtype, check_dtype
+from pyfive.h5py import Datatype, Empty
+from importlib.metadata import version
 
 __version__ = '0.5.0.dev'
+

pyfive/btree.py

@@ -8,6 +8,7 @@
 
 from .core import _padded_size
 from .core import _unpack_struct_from_file
+from .core import _unpack_struct_from
 from .core import Reference
 
 
@@ -440,6 +441,25 @@ def _parse_record(self, record):
         creationorder = struct.unpack_from("<Q", record, 0)[0]
         return {'creationorder': creationorder, 'heapid':record[8:8+7]}
 
+class BTreeV2AttrCreationOrder(BTreeV2):
+    """
+    HDF5 version 2 B-Tree storing attribute creation orders (type 9).
+    See the Type 9 Record Layout, note the different ordering from type 6.
+    """
+    NODE_TYPE = 9
+
+    def _parse_record(self, record):
+        return _unpack_struct_from(V2_BTREE_NODE_TYPE_9_LAYOUT,record)
+
+class BTreeV2AttrNames(BTreeV2):
+    """
+    HDF5 version 2 B-Tree storing attribute names (type 8).
+    """
+    NODE_TYPE = 8  
+
+    def _parse_record(self, record):
+        return _unpack_struct_from(V2_BTREE_NODE_TYPE_8_LAYOUT,record)
+
 
 # IV.A.2.l The Data Storage - Filter Pipeline message
 RESERVED_FILTER = 0
@@ -449,3 +469,20 @@ def _parse_record(self, record):
 SZIP_FILTER = 4
 NBIT_FILTER = 5
 SCALEOFFSET_FILTER = 6
+
+
+# Attribute message B-Tree node types
+# haven't tested type 8 yet, not sure how to get some.
+#
+V2_BTREE_NODE_TYPE_8_LAYOUT = OrderedDict((
+    ('heapid','8s'),
+    ('flags','B'),
+    ('creationorder','I'),
+    ('namehash','I')
+))
+
+V2_BTREE_NODE_TYPE_9_LAYOUT = OrderedDict((
+    ('heapid','8s'),
+    ('flags','B'),
+    ('creationorder','I')
+))

pyfive/core.py

@@ -60,3 +60,5 @@ def _unpack_integer(nbytes, buf, offset=0):
     fmt = "{}s".format(nbytes)
     values = struct.unpack_from(fmt, buf, offset=offset)
     return int.from_bytes(values[0], byteorder="little", signed=False)
+
+