-
Notifications
You must be signed in to change notification settings - Fork 264
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1183 from hmaarrfk/add_set_alignment
Add support for nc_set_alignment and nc_get_alignment
- Loading branch information
Showing
5 changed files
with
224 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| import numpy as np | ||
| from netCDF4 import set_alignment, get_alignment, Dataset | ||
| import netCDF4 | ||
| import os | ||
| import subprocess | ||
| import tempfile | ||
| import unittest | ||
|
|
||
| # During testing, sometimes development versions are used. | ||
| # They may be written as 4.9.1-development | ||
| libversion_no_development = netCDF4.__netcdf4libversion__.split('-')[0] | ||
| libversion = tuple(int(v) for v in libversion_no_development.split('.')) | ||
| has_alignment = (libversion[0] > 4) or ( | ||
| libversion[0] == 4 and (libversion[1] >= 9) | ||
| ) | ||
| try: | ||
| has_h5ls = subprocess.check_call(['h5ls', '--version'], stdout=subprocess.PIPE) == 0 | ||
| except Exception: | ||
| has_h5ls = False | ||
|
|
||
| file_name = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name | ||
|
|
||
|
|
||
| class AlignmentTestCase(unittest.TestCase): | ||
| def setUp(self): | ||
| self.file = file_name | ||
|
|
||
| # This is a global variable in netcdf4, it must be set before File | ||
| # creation | ||
| if has_alignment: | ||
| set_alignment(1024, 4096) | ||
| assert get_alignment() == (1024, 4096) | ||
|
|
||
| f = Dataset(self.file, 'w') | ||
| f.createDimension('x', 4096) | ||
| # Create many datasets so that we decrease the chance of | ||
| # the dataset being randomly aligned | ||
| for i in range(10): | ||
| f.createVariable(f'data{i:02d}', np.float64, ('x',)) | ||
| v = f.variables[f'data{i:02d}'] | ||
| v[...] = 0 | ||
| f.close() | ||
| if has_alignment: | ||
| # ensure to reset the alignment to 1 (default values) so as not to | ||
| # disrupt other tests | ||
| set_alignment(1, 1) | ||
| assert get_alignment() == (1, 1) | ||
|
|
||
| def test_version_settings(self): | ||
| if has_alignment: | ||
| # One should always be able to set the alignment to 1, 1 | ||
| set_alignment(1, 1) | ||
| assert get_alignment() == (1, 1) | ||
| else: | ||
| with self.assertRaises(RuntimeError): | ||
| set_alignment(1, 1) | ||
| with self.assertRaises(RuntimeError): | ||
| get_alignment() | ||
|
|
||
| # if we have no support for alignment, we have no guarantees on | ||
| # how the data can be aligned | ||
| @unittest.skipIf( | ||
| not has_h5ls, | ||
| "h5ls not found." | ||
| ) | ||
| @unittest.skipIf( | ||
| not has_alignment, | ||
| "No support for set_alignment in libnetcdf." | ||
| ) | ||
| def test_setting_alignment(self): | ||
| # We choose to use h5ls instead of h5py since h5ls is very likely | ||
| # to be installed alongside the rest of the tooling required to build | ||
| # netcdf4-python | ||
| # Output from h5ls is expected to look like: | ||
| """ | ||
| Opened "/tmp/tmpqexgozg1.nc" with sec2 driver. | ||
| data00 Dataset {4096/4096} | ||
| Attribute: DIMENSION_LIST {1} | ||
| Type: variable length of | ||
| object reference | ||
| Attribute: _Netcdf4Coordinates {1} | ||
| Type: 32-bit little-endian integer | ||
| Location: 1:563 | ||
| Links: 1 | ||
| Storage: 32768 logical bytes, 32768 allocated bytes, 100.00% utilization | ||
| Type: IEEE 64-bit little-endian float | ||
| Address: 8192 | ||
| data01 Dataset {4096/4096} | ||
| Attribute: DIMENSION_LIST {1} | ||
| Type: variable length of | ||
| object reference | ||
| Attribute: _Netcdf4Coordinates {1} | ||
| Type: 32-bit little-endian integer | ||
| Location: 1:1087 | ||
| Links: 1 | ||
| Storage: 32768 logical bytes, 32768 allocated bytes, 100.00% utilization | ||
| Type: IEEE 64-bit little-endian float | ||
| Address: 40960 | ||
| [...] | ||
| x Dataset {4096/4096} | ||
| Attribute: CLASS scalar | ||
| Type: 16-byte null-terminated ASCII string | ||
| Attribute: NAME scalar | ||
| Type: 64-byte null-terminated ASCII string | ||
| Attribute: REFERENCE_LIST {10} | ||
| Type: struct { | ||
| "dataset" +0 object reference | ||
| "dimension" +8 32-bit little-endian unsigned integer | ||
| } 16 bytes | ||
| Attribute: _Netcdf4Dimid scalar | ||
| Type: 32-bit little-endian integer | ||
| Location: 1:239 | ||
| Links: 1 | ||
| Storage: 16384 logical bytes, 0 allocated bytes | ||
| Type: IEEE 32-bit big-endian float | ||
| Address: 18446744073709551615 | ||
| """ | ||
| h5ls_results = subprocess.check_output( | ||
| ["h5ls", "--verbose", "--address", "--simple", self.file] | ||
| ).decode() | ||
|
|
||
| addresses = { | ||
| f'data{i:02d}': -1 | ||
| for i in range(10) | ||
| } | ||
|
|
||
| data_variable = None | ||
| for line in h5ls_results.split('\n'): | ||
| if not line.startswith(' '): | ||
| data_variable = line.split(' ')[0] | ||
| # only process the data variables we care to inpsect | ||
| if data_variable not in addresses: | ||
| continue | ||
| line = line.strip() | ||
| if line.startswith('Address:'): | ||
| address = int(line.split(':')[1].strip()) | ||
| addresses[data_variable] = address | ||
|
|
||
| for key, address in addresses.items(): | ||
| is_aligned = (address % 4096) == 0 | ||
| assert is_aligned, f"{key} is not aligned. Address = 0x{address:x}" | ||
|
|
||
| # Alternative implementation in h5py | ||
| # import h5py | ||
| # with h5py.File(self.file, 'r') as h5file: | ||
| # for i in range(10): | ||
| # v = h5file[f'data{i:02d}'] | ||
| # assert (dataset.id.get_offset() % 4096) == 0 | ||
|
|
||
| def tearDown(self): | ||
| # Remove the temporary files | ||
| os.remove(self.file) | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| unittest.main() |