Merge pull request #32 from pyiron/subprocess

Implement @conda decorator for executing python functions in specific environments

.ci_support/environment.yml

@@ -3,3 +3,4 @@ channels:
 dependencies:
 - python
 - conda =24.7.1
+- executorlib =0.0.2

.github/workflows/unittests.yml

@@ -48,5 +48,9 @@ jobs:
       run: |
         pip install versioneer[toml]==0.29
         pip install . --no-deps --no-build-isolation
-        conda create -y -n py312 python=3.12.1
+        conda create -y -n py312 python=3.12.1 conda=24.7.1 executorlib=0.0.2
+        conda activate py312
+        pip install versioneer[toml]==0.29
+        pip install . --no-deps --no-build-isolation
+        conda deactivate
         python -m unittest discover tests

README.md

@@ -7,6 +7,7 @@ Create a new conda environment - in this example a conda environment for Python
 conda create -n py312 python=3.12 
 ```
 
+### Subprocess Interface
 Open a python shell in your base environment where `conda_subprocess` is installed and execute `python --version` in the
 `py312` environment:
 ```python
@@ -64,6 +65,20 @@ process.communicate()
 >>> (b'Python 3.12.1\n', None)
 ```
 
+### Decorator 
+In analogy to the subprocess interface the `conda_subprocess` also introduces the `@conda` decorator to 
+execute python functions in a separate conda environment:
+```python
+from conda_subprocess.decorator import conda
+
+@conda(prefix_name="py312")
+def add_function(parameter_1, parameter_2):
+    return parameter_1 + parameter_2
+
+add_function(parameter_1=1, parameter_2=2)
+>>> 3
+```
+
 ## Remarks
 * The `shell` parameter and the `env` parameter are not supported in `Popen()` and all derived methods. 
 * The `pipesize` parameter and the `process_group` parameter were removed for compatibility with python 3.9. 

conda_subprocess/decorator.py

@@ -0,0 +1,50 @@
+import subprocess
+from concurrent.futures import Future
+from socket import gethostname
+from typing import Optional
+
+from executorlib.shared.communication import SocketInterface
+from executorlib.shared.executor import get_command_path
+from executorlib.shared.interface import SubprocessInterface
+
+from conda_subprocess.process import Popen
+
+
+def conda(
+    prefix_name: Optional[str] = None,
+    prefix_path: Optional[str] = None,
+    hostname_localhost: bool = False,
+):
+    def conda_function(funct):
+        def function_wrapped(*args, **kwargs):
+            task_future = Future()
+            task_dict = {
+                "fn": funct,
+                "args": args,
+                "kwargs": kwargs,
+                "resource_dict": {"cores": 1},
+            }
+            interface = SocketInterface(interface=SubprocessInterface(cores=1))
+            command_lst = [
+                "python",
+                get_command_path(executable="interactive_serial.py"),
+            ]
+            if not hostname_localhost:
+                command_lst += ["--host", gethostname()]
+            command_lst += ["--zmqport", str(interface.bind_to_random_port())]
+            interface._interface._process = Popen(
+                args=interface._interface.generate_command(command_lst=command_lst),
+                cwd=interface._interface._cwd,
+                stdin=subprocess.DEVNULL,
+                prefix_name=prefix_name,
+                prefix_path=prefix_path,
+            )
+            task_future.set_result(
+                interface.send_and_receive_dict(input_dict=task_dict)
+            )
+            interface.shutdown(wait=True)
+            return task_future.result()
+
+        return function_wrapped
+
+    return conda_function

pyproject.toml

@@ -31,6 +31,11 @@ dynamic = ["version"]
 [project.urls]
 Repository = "https://github.com/pyiron/conda_subprocess"
 
+[project.optional-dependencies]
+executorlib = [
+    "executorlib==0.0.2",
+]
+
 [tool.setuptools.packages.find]
 include = ["conda_subprocess*"]
 

tests/test_conda_function.py

@@ -0,0 +1,35 @@
+import sys
+import unittest
+
+try:
+    from conda_subprocess.decorator import conda
+    from executorlib.shared.executor import cloudpickle_register
+except ImportError:
+
+    def conda(prefix_name=None, prefix_path=None):
+        def wrap_function(funct):
+            def function_out(*args, **kwargs):
+                return None
+
+            return function_out
+
+        return wrap_function
+
+
+@conda(prefix_name="py312")
+def add_function(parameter_1, parameter_2):
+    import os
+
+    return (parameter_1 + parameter_2, os.environ["CONDA_PREFIX"])
+
+
+@unittest.skipIf(
+    sys.version_info.minor != 12,
+    "Test environment has to be Python 3.12 for consistency.",
+)
+class TestCondaFunction(unittest.TestCase):
+    def test_conda_function(self):
+        cloudpickle_register(ind=1)
+        number, prefix = add_function(parameter_1=1, parameter_2=2)
+        self.assertEqual(prefix[-5:], "py312")
+        self.assertEqual(number, 3)