From 7f4fd38b302f2e6437b391fe6f7f855690872e72 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 10 Oct 2024 09:55:34 +0100 Subject: [PATCH 1/5] Add fdb_home argument --- src/pyfdb/pyfdb.py | 35 ++++++++++++++++- tests/test_passing_config_directly.py | 54 +++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/pyfdb/pyfdb.py b/src/pyfdb/pyfdb.py index 3f22169..b811916 100644 --- a/src/pyfdb/pyfdb.py +++ b/src/pyfdb/pyfdb.py @@ -290,15 +290,48 @@ class FDB: fdb = pyfdb.FDB() # call fdb.archive, fdb.list, fdb.retrieve, fdb.flush as needed. + # Construct with a particular FDB_HOME + fdb = pyfdb.FDB(fdb_home="/path/to/fdb_home") + + # Construct from a config dictionary + config = dict( + type="local", + engine="toc", + schema="path/to/schema", + spaces=[ + dict( + handler="Default", + roots=[ + {"path": "/path/to/root"}, + ], + ) + ], + ) + fdb = pyfdb.FDB(config) + See the module level pyfdb.list, pyfdb.retrieve, and pyfdb.archive docstrings for more information on these functions. """ __fdb = None - def __init__(self, config=None, user_config=None): + def __init__( + self, + config=None, + user_config=None, + fdb_home=None, + ): + """ + Args: + """ fdb = ffi.new("fdb_handle_t**") + if fdb_home is not None and config is not None: + raise ValueError("Cannot specify both fdb_home and config") + + if fdb_home is not None: + config = dict(fdb_home=str(fdb_home)) + if config is not None or user_config is not None: def prepare_config(c): diff --git a/tests/test_passing_config_directly.py b/tests/test_passing_config_directly.py index fcb947e..a89cf52 100644 --- a/tests/test_passing_config_directly.py +++ b/tests/test_passing_config_directly.py @@ -1,9 +1,63 @@ +import shutil from pathlib import Path from tempfile import TemporaryDirectory +from pytest import raises + import pyfdb +def test_conflicting_arguments(): + "You can't pass both fdb_home and config because the former overrides the latter" + assert raises(ValueError, pyfdb.FDB, fdb_home="/tmp", config={}) + + +def test_fdb_home(): + with TemporaryDirectory() as tmp_home: + tests_dir = Path(__file__).parent + schema_path = tests_dir / "default_fdb_schema" + + home = Path(tmp_home) + etc = home / "etc" / "fdb" + root_path = home / "root" + root_path.mkdir(parents=True) + etc.mkdir(parents=True) + + shutil.copy(schema_path, etc / "schema") + + with open(etc / "config.yaml", "w") as f: + f.write( + f""" +--- +type: local +engine: toc +schema: {etc / "schema"} +spaces: +- handler: Default + roots: + - path: "{root_path}" + writable: true + visit: true +""" + ) + + with open(etc / "config.yaml", "r") as f: + print(f.read()) + + fdb = pyfdb.FDB(fdb_home=home) + data = open(tests_dir / "x138-300.grib", "rb").read() + fdb.archive(data) + fdb.flush() + + list_output = list(fdb.list(keys=True)) + assert len(list_output) == 1 + + # Check that the archive path is in the tmp directory + # On OSX tmp file paths look like /private/var/folders/.../T/tmp.../x138-300.grib + # While the tmp directory looks like /var/folders/.../T/tmp.../ hence why this check is not "startwith" + assert tmp_home in list_output[0]["path"] + + def test_direct_config(): with TemporaryDirectory() as tmp_root: tests_dir = Path(__file__).parent From 685861a05a4b1b6c1f34a4481eeb474d85ff3771 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 10 Oct 2024 10:38:53 +0100 Subject: [PATCH 2/5] Add types --- src/pyfdb/pyfdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pyfdb/pyfdb.py b/src/pyfdb/pyfdb.py index b811916..e8b25c5 100644 --- a/src/pyfdb/pyfdb.py +++ b/src/pyfdb/pyfdb.py @@ -16,6 +16,8 @@ import json import os from functools import wraps +from pathlib import Path +from typing import Optional, Union import cffi import findlibs @@ -317,9 +319,9 @@ class FDB: def __init__( self, - config=None, - user_config=None, - fdb_home=None, + config: Optional[dict] = None, + user_config: Optional[dict] = None, + fdb_home: Union[str, Path, None] = None, ): """ Args: From c178dbfa0016b924e6cb10b44cf1657549fd3a66 Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 18 Oct 2024 11:52:18 +0100 Subject: [PATCH 3/5] Try adding resolve to paths to see if it fixes tmp symlink issues --- tests/test_passing_config_directly.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_passing_config_directly.py b/tests/test_passing_config_directly.py index a89cf52..cf1a89a 100644 --- a/tests/test_passing_config_directly.py +++ b/tests/test_passing_config_directly.py @@ -55,7 +55,7 @@ def test_fdb_home(): # Check that the archive path is in the tmp directory # On OSX tmp file paths look like /private/var/folders/.../T/tmp.../x138-300.grib # While the tmp directory looks like /var/folders/.../T/tmp.../ hence why this check is not "startwith" - assert tmp_home in list_output[0]["path"] + assert str(Path(tmp_home).resolve()) in str(Path(list_output[0]["path"]).resolve()) def test_direct_config(): From 797e01960de90a34baf84cfc2e5c95c825b0e80b Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 8 Nov 2024 11:20:47 +0000 Subject: [PATCH 4/5] Fix tox for use with eccodes --- pyproject.toml | 2 +- tests/test_passing_config_directly.py | 21 +++++++++++++++------ tox.ini | 6 +++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 631cf48..b663de8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ urls.Homepage = "https://github.com/ecmwf/pyfdb" urls.Issues = "https://github.com/ecmwf/pyfdb/issues" urls.Repository = "https://github.com/ecmwf/pyfdb" -requires-python = ">=3.9" +requires-python = ">=3.8" dependencies = [ "cffi", diff --git a/tests/test_passing_config_directly.py b/tests/test_passing_config_directly.py index cf1a89a..3242a72 100644 --- a/tests/test_passing_config_directly.py +++ b/tests/test_passing_config_directly.py @@ -55,7 +55,13 @@ def test_fdb_home(): # Check that the archive path is in the tmp directory # On OSX tmp file paths look like /private/var/folders/.../T/tmp.../x138-300.grib # While the tmp directory looks like /var/folders/.../T/tmp.../ hence why this check is not "startwith" - assert str(Path(tmp_home).resolve()) in str(Path(list_output[0]["path"]).resolve()) + + # Disabled for now because the HPC has a different convention and I don't know how to deal with all cases + # On the HPC + # tmp_home looks like '/etc/ecmwf/ssd/ssd1/tmpdirs/***.31103395/tmpg137a4ml' + # while the archive path looks like '/etc/ecmwf/ssd/ssd1/tmpdirs/***.31103395/data/fdb/...' + + # assert str(Path(tmp_home).resolve()) in str(Path(list_output[0]["path"]).resolve()) def test_direct_config(): @@ -87,7 +93,9 @@ def test_direct_config(): # Check that the archive path is in the tmp directory # On OSX tmp file paths look like /private/var/folders/.../T/tmp.../x138-300.grib # While the tmp directory looks like /var/folders/.../T/tmp.../ hence why this check is not "startwith" - assert tmp_root in list_output[0]["path"] + + # Disabled for now because the HPC has a different convention and I don't know how to deal with all cases + # assert tmp_root in list_output[0]["path"] def test_opening_two_fdbs(): @@ -131,7 +139,8 @@ def test_opening_two_fdbs(): fdb.archive(data) fdb.flush() - for fdb, root in [(fdb1, tmp_root1), (fdb2, tmp_root2)]: - list_output = list(fdb.list(keys=True)) - assert len(list_output) == 1 - assert root in list_output[0]["path"] + # Disabled for now + # for fdb, root in [(fdb1, tmp_root1), (fdb2, tmp_root2)]: + # list_output = list(fdb.list(keys=True)) + # assert len(list_output) == 1 + # assert root in list_output[0]["path"] diff --git a/tox.ini b/tox.ini index d93e522..337019d 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,10 @@ wheel_build_env = .pkg deps = pytest>=6 pass_env = - FDB_HOME + FDB_HOME # (optional) Tells fdb where to look for config files + FDB5_HOME # (optional) Tell findlibs where to find lib/libfdb5.dylib + ECCODES_HOME # (optional) Tell findlibs where to find lib/libeccodes.dylib + ECCODES_PYTHON_TRACE_LIB_SEARCH # (optional) Print debug info about eccodes search for libeccodes.dylib + ECCODES_PYTHON_USE_FINDLIBS # (optional) Force eccodes to use findlibs to find libeccodes.dylib rather that using the binary installed by pip commands = pytest {tty:--color=yes} {posargs} From 28eed3964bdba1c40efd1c8d5131b6483f01acd0 Mon Sep 17 00:00:00 2001 From: Tobias Kremer Date: Fri, 8 Nov 2024 15:20:40 +0100 Subject: [PATCH 5/5] Introduce ATOS flag for temp directory generation --- tests/test_passing_config_directly.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/test_passing_config_directly.py b/tests/test_passing_config_directly.py index 3242a72..7c221f7 100644 --- a/tests/test_passing_config_directly.py +++ b/tests/test_passing_config_directly.py @@ -1,3 +1,4 @@ +import os import shutil from pathlib import Path from tempfile import TemporaryDirectory @@ -6,14 +7,16 @@ import pyfdb +IS_ATOS = os.environ.get("SCRATCHDIR", None) + def test_conflicting_arguments(): - "You can't pass both fdb_home and config because the former overrides the latter" + "You can't pass both fdb_home and config because the former overrides the latter!" assert raises(ValueError, pyfdb.FDB, fdb_home="/tmp", config={}) def test_fdb_home(): - with TemporaryDirectory() as tmp_home: + with TemporaryDirectory(dir=IS_ATOS) as tmp_home: tests_dir = Path(__file__).parent schema_path = tests_dir / "default_fdb_schema" @@ -61,11 +64,11 @@ def test_fdb_home(): # tmp_home looks like '/etc/ecmwf/ssd/ssd1/tmpdirs/***.31103395/tmpg137a4ml' # while the archive path looks like '/etc/ecmwf/ssd/ssd1/tmpdirs/***.31103395/data/fdb/...' - # assert str(Path(tmp_home).resolve()) in str(Path(list_output[0]["path"]).resolve()) + assert os.path.realpath(tmp_home) in os.path.realpath(list_output[0]["path"]) def test_direct_config(): - with TemporaryDirectory() as tmp_root: + with TemporaryDirectory(dir=IS_ATOS) as tmp_root: tests_dir = Path(__file__).parent config = dict( @@ -95,11 +98,11 @@ def test_direct_config(): # While the tmp directory looks like /var/folders/.../T/tmp.../ hence why this check is not "startwith" # Disabled for now because the HPC has a different convention and I don't know how to deal with all cases - # assert tmp_root in list_output[0]["path"] + assert os.path.realpath(tmp_root) in os.path.realpath(list_output[0]["path"]) def test_opening_two_fdbs(): - with TemporaryDirectory() as tmp_root1, TemporaryDirectory() as tmp_root2: + with TemporaryDirectory(dir=IS_ATOS) as tmp_root1, TemporaryDirectory(dir=IS_ATOS) as tmp_root2: tests_dir = Path(__file__).parent fdb1 = pyfdb.FDB( @@ -139,8 +142,7 @@ def test_opening_two_fdbs(): fdb.archive(data) fdb.flush() - # Disabled for now - # for fdb, root in [(fdb1, tmp_root1), (fdb2, tmp_root2)]: - # list_output = list(fdb.list(keys=True)) - # assert len(list_output) == 1 - # assert root in list_output[0]["path"] + for fdb, root in [(fdb1, tmp_root1), (fdb2, tmp_root2)]: + list_output = list(fdb.list(keys=True)) + assert len(list_output) == 1 + assert os.path.realpath(root) in os.path.realpath(list_output[0]["path"])