Merge pull request #433 from pyiron/hostname_localhost

Automatically set hostname_localhost

Author: jan-janssen

executorlib/__init__.py

@@ -103,7 +103,7 @@ def __init__(
         flux_executor=None,
         flux_executor_pmi_mode: Optional[str] = None,
         flux_executor_nesting: bool = False,
-        hostname_localhost: bool = False,
+        hostname_localhost: Optional[bool] = None,
         block_allocation: bool = True,
         init_function: Optional[callable] = None,
         disable_dependencies: bool = False,
@@ -127,7 +127,7 @@ def __new__(
         flux_executor=None,
         flux_executor_pmi_mode: Optional[str] = None,
         flux_executor_nesting: bool = False,
-        hostname_localhost: bool = False,
+        hostname_localhost: Optional[bool] = None,
         block_allocation: bool = True,
         init_function: Optional[callable] = None,
         disable_dependencies: bool = False,

executorlib/interactive/__init__.py

@@ -40,7 +40,7 @@ def create_executor(
     flux_executor=None,
     flux_executor_pmi_mode: Optional[str] = None,
     flux_executor_nesting: bool = False,
-    hostname_localhost: bool = False,
+    hostname_localhost: Optional[bool] = None,
     block_allocation: bool = False,
     init_function: Optional[callable] = None,
 ):

executorlib/shared/communication.py

@@ -1,4 +1,6 @@
+import sys
 from socket import gethostname
+from typing import Optional
 
 import cloudpickle
 import zmq
@@ -119,7 +121,7 @@ def __del__(self):
 def interface_bootup(
     command_lst: list[str],
     connections,
-    hostname_localhost: bool = False,
+    hostname_localhost: Optional[bool] = None,
 ):
     """
     Start interface for ZMQ communication
@@ -139,6 +141,10 @@ def interface_bootup(
     Returns:
          executorlib.shared.communication.SocketInterface: socket interface for zmq communication
     """
+    if hostname_localhost is None and sys.platform == "darwin":
+        hostname_localhost = True
+    elif hostname_localhost is None:
+        hostname_localhost = False
     if not hostname_localhost:
         command_lst += [
             "--host",

executorlib/shared/executor.py

@@ -302,7 +302,7 @@ def execute_parallel_tasks(
     future_queue: queue.Queue,
     cores: int = 1,
     spawner: BaseSpawner = MpiExecSpawner,
-    hostname_localhost: bool = False,
+    hostname_localhost: Optional[bool] = None,
     init_function: Optional[Callable] = None,
     **kwargs,
 ) -> None:
@@ -358,7 +358,7 @@ def execute_separate_tasks(
     future_queue: queue.Queue,
     spawner: BaseSpawner = MpiExecSpawner,
     max_cores: int = 1,
-    hostname_localhost: bool = False,
+    hostname_localhost: Optional[bool] = None,
     **kwargs,
 ):
     """
@@ -605,7 +605,7 @@ def _submit_function_to_separate_process(
     spawner: BaseSpawner,
     executor_kwargs: dict,
     max_cores: int = 1,
-    hostname_localhost: bool = False,
+    hostname_localhost: Optional[bool] = None,
 ):
     """
     Submit function to be executed in separate Python process

notebooks/examples.ipynb

@@ -221,7 +221,7 @@
     "        flux_executor=flux_exe,\n",
     "        flux_executor_pmi_mode=None,\n",
     "        flux_executor_nesting=False,\n",
-    "        hostname_localhost=False,  # only required on MacOS\n",
+    "        hostname_localhost=None,  # only required on MacOS\n",
     "        block_allocation=False,  # reuse existing processes with fixed resources\n",
     "        init_function=None,  # only available with block_allocation=True\n",
     "        disable_dependencies=False,  # disable dependency check for faster execution\n",
@@ -243,7 +243,7 @@
     "                \"flux_executor\": flux_exe,\n",
     "                \"flux_executor_pmi_mode\": None,\n",
     "                \"flux_executor_nesting\": False,\n",
-    "                \"hostname_localhost\": False,  # only required on MacOS\n",
+    "                \"hostname_localhost\": None,  # only required on MacOS\n",
     "            },\n",
     "        )\n",
     "        print(future_obj.result())"
@@ -749,8 +749,16 @@
     "Workstations, especially workstations with MacOs can have rather strict firewall settings. This includes limiting the\n",
     "look up of hostnames and communicating with itself via their own hostname. To directly connect to `localhost` rather\n",
     "than using the hostname which is the default for distributed systems, the `hostname_localhost=True` parameter is \n",
-    "introduced. "
+    "introduced."
    ]
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "601852447d3839c4"
   }
  ],
  "metadata": {

tests/test_dependencies_executor.py

@@ -41,7 +41,7 @@ def merge(lst):
 
 class TestExecutorWithDependencies(unittest.TestCase):
     def test_executor(self):
-        with Executor(max_cores=1, backend="local", hostname_localhost=True) as exe:
+        with Executor(max_cores=1, backend="local") as exe:
             cloudpickle_register(ind=1)
             future_1 = exe.submit(add_function, 1, parameter_2=2)
             future_2 = exe.submit(add_function, 1, parameter_2=future_1)
@@ -55,7 +55,6 @@ def test_executor_dependency_plot(self):
         with Executor(
             max_cores=1,
             backend="local",
-            hostname_localhost=True,
             plot_dependency_graph=True,
         ) as exe:
             cloudpickle_register(ind=1)
@@ -105,7 +104,6 @@ def test_dependency_steps(self):
             gpus_per_worker=0,
             openmpi_oversubscribe=False,
             backend="local",
-            hostname_localhost=True,
         )
         process = RaisingThread(
             target=execute_tasks_with_dependencies,
@@ -127,7 +125,7 @@ def test_dependency_steps(self):
     def test_many_to_one(self):
         length = 5
         parameter = 1
-        with Executor(max_cores=2, backend="local", hostname_localhost=True) as exe:
+        with Executor(max_cores=2, backend="local") as exe:
             cloudpickle_register(ind=1)
             future_lst = exe.submit(
                 generate_tasks,
@@ -162,7 +160,6 @@ def test_many_to_one_plot(self):
         with Executor(
             max_cores=2,
             backend="local",
-            hostname_localhost=True,
             plot_dependency_graph=True,
         ) as exe:
             cloudpickle_register(ind=1)

tests/test_executor_backend_mpi.py

@@ -22,9 +22,7 @@ def mpi_funct(i):
 
 class TestExecutorBackend(unittest.TestCase):
     def test_meta_executor_serial(self):
-        with Executor(
-            max_cores=2, hostname_localhost=True, backend="local", block_allocation=True
-        ) as exe:
+        with Executor(max_cores=2, backend="local", block_allocation=True) as exe:
             cloudpickle_register(ind=1)
             fs_1 = exe.submit(calc, 1)
             fs_2 = exe.submit(calc, 2)
@@ -34,9 +32,7 @@ def test_meta_executor_serial(self):
             self.assertTrue(fs_2.done())
 
     def test_meta_executor_single(self):
-        with Executor(
-            max_cores=1, hostname_localhost=True, backend="local", block_allocation=True
-        ) as exe:
+        with Executor(max_cores=1, backend="local", block_allocation=True) as exe:
             cloudpickle_register(ind=1)
             fs_1 = exe.submit(calc, 1)
             fs_2 = exe.submit(calc, 2)
@@ -52,7 +48,6 @@ def test_meta_executor_parallel(self):
         with Executor(
             max_workers=2,
             cores_per_worker=2,
-            hostname_localhost=True,
             backend="local",
             block_allocation=True,
         ) as exe:
@@ -67,14 +62,12 @@ def test_errors(self):
                 max_cores=1,
                 cores_per_worker=1,
                 threads_per_core=2,
-                hostname_localhost=True,
                 backend="local",
             )
         with self.assertRaises(TypeError):
             Executor(
                 max_cores=1,
                 cores_per_worker=1,
                 gpus_per_worker=1,
-                hostname_localhost=True,
                 backend="local",
             )

tests/test_executor_backend_mpi_noblock.py

@@ -16,7 +16,6 @@ class TestExecutorBackend(unittest.TestCase):
     def test_meta_executor_serial(self):
         with Executor(
             max_cores=2,
-            hostname_localhost=True,
             backend="local",
             block_allocation=False,
         ) as exe:
@@ -31,7 +30,6 @@ def test_meta_executor_serial(self):
     def test_meta_executor_single(self):
         with Executor(
             max_cores=1,
-            hostname_localhost=True,
             backend="local",
             block_allocation=False,
         ) as exe:
@@ -49,29 +47,25 @@ def test_errors(self):
                 max_cores=1,
                 cores_per_worker=1,
                 threads_per_core=2,
-                hostname_localhost=True,
                 backend="local",
             )
         with self.assertRaises(TypeError):
             Executor(
                 max_cores=1,
                 cores_per_worker=1,
                 gpus_per_worker=1,
-                hostname_localhost=True,
                 backend="local",
             )
         with self.assertRaises(ValueError):
             with Executor(
                 max_cores=1,
-                hostname_localhost=True,
                 backend="local",
                 block_allocation=False,
             ) as exe:
                 exe.submit(resource_dict, resource_dict={})
         with self.assertRaises(ValueError):
             with Executor(
                 max_cores=1,
-                hostname_localhost=True,
                 backend="local",
                 block_allocation=True,
             ) as exe:

tests/test_integration_pyiron_workflow.py

@@ -74,7 +74,7 @@ def slowly_returns_dynamic(dynamic_arg):
             return dynamic_arg
 
         dynamic_dynamic = slowly_returns_dynamic()
-        executor = Executor(hostname_localhost=True, block_allocation=True)
+        executor = Executor(block_allocation=True)
         cloudpickle_register(ind=1)
         dynamic_object = does_nothing()
         fs = executor.submit(dynamic_dynamic.run, dynamic_object)
@@ -104,7 +104,7 @@ def slowly_returns_42():
         self.assertIsNone(
             dynamic_42.result, msg="Just a sanity check that the test is set up right"
         )
-        executor = Executor(hostname_localhost=True, block_allocation=True)
+        executor = Executor(block_allocation=True)
         cloudpickle_register(ind=1)
         fs = executor.submit(dynamic_42.run)
         fs.add_done_callback(dynamic_42.process_result)
@@ -135,7 +135,7 @@ def returns_42():
             dynamic_42.running,
             msg="Sanity check that the test starts in the expected condition",
         )
-        executor = Executor(hostname_localhost=True, block_allocation=True)
+        executor = Executor(block_allocation=True)
         cloudpickle_register(ind=1)
         fs = executor.submit(dynamic_42.run)
         fs.add_done_callback(dynamic_42.process_result)
@@ -159,7 +159,7 @@ def raise_error():
             raise RuntimeError
 
         re = raise_error()
-        executor = Executor(hostname_localhost=True, block_allocation=True)
+        executor = Executor(block_allocation=True)
         cloudpickle_register(ind=1)
         fs = executor.submit(re.run)
         with self.assertRaises(
@@ -189,7 +189,7 @@ def slowly_returns_dynamic():
             return inside_variable
 
         dynamic_dynamic = slowly_returns_dynamic()
-        executor = Executor(hostname_localhost=True, block_allocation=True)
+        executor = Executor(block_allocation=True)
         cloudpickle_register(ind=1)
         fs = executor.submit(dynamic_dynamic.run)
         self.assertIsInstance(
@@ -218,7 +218,7 @@ def slow():
             return fortytwo
 
         f = slow()
-        executor = Executor(hostname_localhost=True, block_allocation=True)
+        executor = Executor(block_allocation=True)
         cloudpickle_register(ind=1)
         fs = executor.submit(f.run)
         self.assertEqual(