From 29d68fef91df2d26274a2f104835c9ffd1aab5f9 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 29 Oct 2024 11:20:02 +0100 Subject: [PATCH 1/3] Add additional input checks --- executorlib/__init__.py | 5 ++++ executorlib/cache/executor.py | 22 +++++------------- executorlib/standalone/inputcheck.py | 34 ++++++++++++++++++++++++++++ tests/test_shared_input_check.py | 26 +++++++++++++++++++++ 4 files changed, 71 insertions(+), 16 deletions(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 159031a3..40788f2d 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -8,6 +8,9 @@ from executorlib.standalone.inputcheck import ( check_refresh_rate as _check_refresh_rate, ) +from executorlib.standalone.inputcheck import ( + check_pysqa_config_directory as _check_pysqa_config_directory +) __version__ = _get_versions()["version"] __all__ = [] @@ -194,6 +197,7 @@ def __new__( init_function=init_function, ) elif not disable_dependencies: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) return ExecutorWithDependencies( max_workers=max_workers, backend=backend, @@ -210,6 +214,7 @@ def __new__( plot_dependency_graph=plot_dependency_graph, ) else: + _check_pysqa_config_directory(pysqa_config_directory=pysqa_config_directory) _check_plot_dependency_graph(plot_dependency_graph=plot_dependency_graph) _check_refresh_rate(refresh_rate=refresh_rate) return create_executor( diff --git a/executorlib/cache/executor.py b/executorlib/cache/executor.py index 271a1f2e..7cf8cd85 100644 --- a/executorlib/cache/executor.py +++ b/executorlib/cache/executor.py @@ -10,6 +10,9 @@ from executorlib.standalone.inputcheck import ( check_executor, check_nested_flux_executor, + check_flux_executor_pmi_mode, + check_max_workers_and_cores, + check_hostname_localhost, ) from executorlib.standalone.thread import RaisingThread @@ -89,18 +92,6 @@ def create_file_executor( ): if cache_directory is None: cache_directory = "executorlib_cache" - if max_workers != 1: - raise ValueError( - "The number of workers cannot be controlled with the pysqa based backend." - ) - if max_cores != 1: - raise ValueError( - "The number of cores cannot be controlled with the pysqa based backend." - ) - if hostname_localhost is not None: - raise ValueError( - "The option to connect to hosts based on their hostname is not available with the pysqa based backend." - ) if block_allocation: raise ValueError( "The option block_allocation is not available with the pysqa based backend." @@ -109,10 +100,9 @@ def create_file_executor( raise ValueError( "The option to specify an init_function is not available with the pysqa based backend." ) - if flux_executor_pmi_mode is not None: - raise ValueError( - "The option to specify the flux pmi mode is not available with the pysqa based backend." - ) + check_flux_executor_pmi_mode(flux_executor_pmi_mode=flux_executor_pmi_mode) + check_max_workers_and_cores(max_cores=max_cores, max_workers=max_workers) + check_hostname_localhost(hostname_localhost=hostname_localhost) check_executor(executor=flux_executor) check_nested_flux_executor(nested_flux_executor=flux_executor_nesting) return FileExecutor( diff --git a/executorlib/standalone/inputcheck.py b/executorlib/standalone/inputcheck.py index d1d9800f..5fd33202 100644 --- a/executorlib/standalone/inputcheck.py +++ b/executorlib/standalone/inputcheck.py @@ -131,6 +131,40 @@ def check_init_function(block_allocation: bool, init_function: Callable) -> None raise ValueError("") +def check_max_workers_and_cores(max_workers: int, max_cores: int) -> None: + if max_workers != 1: + raise ValueError( + "The number of workers cannot be controlled with the pysqa based backend." + ) + if max_cores != 1: + raise ValueError( + "The number of cores cannot be controlled with the pysqa based backend." + ) + +def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None: + if hostname_localhost is not None: + raise ValueError( + "The option to connect to hosts based on their hostname is not available with the pysqa based backend." + ) + + +def check_flux_executor_pmi_mode(flux_executor_pmi_mode: Optional[str]) -> None: + if flux_executor_pmi_mode is not None: + raise ValueError( + "The option to specify the flux pmi mode is not available with the pysqa based backend." + ) + + +def check_pysqa_config_directory(pysqa_config_directory: Optional[str]) -> None: + """ + Check if pysqa_config_directory is None and raise a ValueError if it is not. + """ + if pysqa_config_directory is not None: + raise ValueError( + "pysqa_config_directory parameter is only supported for pysqa backend." + ) + + def validate_number_of_cores(max_cores: int, max_workers: int) -> int: """ Validate the number of cores and return the appropriate value. diff --git a/tests/test_shared_input_check.py b/tests/test_shared_input_check.py index 4444dc48..b4b4df84 100644 --- a/tests/test_shared_input_check.py +++ b/tests/test_shared_input_check.py @@ -13,6 +13,10 @@ check_refresh_rate, check_resource_dict, check_resource_dict_is_empty, + check_flux_executor_pmi_mode, + check_max_workers_and_cores, + check_hostname_localhost, + check_pysqa_config_directory, ) @@ -69,3 +73,25 @@ def test_check_nested_flux_executor(self): def test_check_plot_dependency_graph(self): with self.assertRaises(ValueError): check_plot_dependency_graph(plot_dependency_graph=True) + + def test_check_flux_executor_pmi_mode(self): + with self.assertRaises(ValueError): + check_flux_executor_pmi_mode(flux_executor_pmi_mode="test") + + def test_check_max_workers_and_cores(self): + with self.assertRaises(ValueError): + check_max_workers_and_cores(max_workers=2, max_cores=1) + with self.assertRaises(ValueError): + check_max_workers_and_cores(max_workers=1, max_cores=2) + with self.assertRaises(ValueError): + check_max_workers_and_cores(max_workers=2, max_cores=2) + + def test_check_hostname_localhost(self): + with self.assertRaises(ValueError): + check_hostname_localhost(hostname_localhost=True) + with self.assertRaises(ValueError): + check_hostname_localhost(hostname_localhost=True) + + def test_check_pysqa_config_directory(self): + with self.assertRaises(ValueError): + check_pysqa_config_directory(pysqa_config_directory="path/to/config") From da0db2533eef49bf4ee30c038df48c10a08b5e5e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:20:50 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/__init__.py | 4 ++-- executorlib/cache/executor.py | 4 ++-- executorlib/standalone/inputcheck.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 40788f2d..bab1ffc5 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -6,10 +6,10 @@ check_plot_dependency_graph as _check_plot_dependency_graph, ) from executorlib.standalone.inputcheck import ( - check_refresh_rate as _check_refresh_rate, + check_pysqa_config_directory as _check_pysqa_config_directory, ) from executorlib.standalone.inputcheck import ( - check_pysqa_config_directory as _check_pysqa_config_directory + check_refresh_rate as _check_refresh_rate, ) __version__ = _get_versions()["version"] diff --git a/executorlib/cache/executor.py b/executorlib/cache/executor.py index 7cf8cd85..cf750780 100644 --- a/executorlib/cache/executor.py +++ b/executorlib/cache/executor.py @@ -9,10 +9,10 @@ ) from executorlib.standalone.inputcheck import ( check_executor, - check_nested_flux_executor, check_flux_executor_pmi_mode, - check_max_workers_and_cores, check_hostname_localhost, + check_max_workers_and_cores, + check_nested_flux_executor, ) from executorlib.standalone.thread import RaisingThread diff --git a/executorlib/standalone/inputcheck.py b/executorlib/standalone/inputcheck.py index 5fd33202..9a466265 100644 --- a/executorlib/standalone/inputcheck.py +++ b/executorlib/standalone/inputcheck.py @@ -141,6 +141,7 @@ def check_max_workers_and_cores(max_workers: int, max_cores: int) -> None: "The number of cores cannot be controlled with the pysqa based backend." ) + def check_hostname_localhost(hostname_localhost: Optional[bool]) -> None: if hostname_localhost is not None: raise ValueError( From 375721e2f93c643f3eb5a5ca5657db90fd1d5a61 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 29 Oct 2024 11:21:35 +0100 Subject: [PATCH 3/3] Update test_shared_input_check.py --- tests/test_shared_input_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_shared_input_check.py b/tests/test_shared_input_check.py index b4b4df84..8899a75e 100644 --- a/tests/test_shared_input_check.py +++ b/tests/test_shared_input_check.py @@ -90,7 +90,7 @@ def test_check_hostname_localhost(self): with self.assertRaises(ValueError): check_hostname_localhost(hostname_localhost=True) with self.assertRaises(ValueError): - check_hostname_localhost(hostname_localhost=True) + check_hostname_localhost(hostname_localhost=False) def test_check_pysqa_config_directory(self): with self.assertRaises(ValueError):