From 9dd336735b2176a294e9b972eb4e09ae3da69e27 Mon Sep 17 00:00:00 2001 From: Ruiyang Wang Date: Fri, 24 May 2024 18:23:12 -0700 Subject: [PATCH 1/5] allow full path in conda rt env Signed-off-by: Ruiyang Wang --- python/ray/_private/runtime_env/conda.py | 13 ++- .../ray/_private/runtime_env/conda_utils.py | 29 +++++- .../ray/tests/test_runtime_env_complicated.py | 95 ++++++++++++++++++- 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/python/ray/_private/runtime_env/conda.py b/python/ray/_private/runtime_env/conda.py index a28cc02cb046..12022b14c858 100644 --- a/python/ray/_private/runtime_env/conda.py +++ b/python/ray/_private/runtime_env/conda.py @@ -18,7 +18,8 @@ create_conda_env_if_needed, delete_conda_env, get_conda_activate_commands, - get_conda_env_list, + get_conda_info_json, + get_conda_envs, ) from ray._private.runtime_env.context import RuntimeEnvContext from ray._private.runtime_env.packaging import Protocol, parse_uri @@ -342,13 +343,15 @@ def _create(): if result in self._validated_named_conda_env: return 0 - conda_env_list = get_conda_env_list() - envs = [Path(env).name for env in conda_env_list] - if result not in envs: + conda_info = get_conda_info_json() + envs = get_conda_envs(conda_info) + + # We accept `result` as a conda name or full path. + if not any(result == env[0] or result == env[1] for env in envs): raise ValueError( f"The given conda environment '{result}' " f"from the runtime env {runtime_env} doesn't " - "exist from the output of `conda env list --json`. " + "exist from the output of `conda info --json`. " "You can only specify an env that already exists. " f"Please make sure to create an env {result} " ) diff --git a/python/ray/_private/runtime_env/conda_utils.py b/python/ray/_private/runtime_env/conda_utils.py index 757ca4a998dd..a33916c5e46f 100644 --- a/python/ray/_private/runtime_env/conda_utils.py +++ b/python/ray/_private/runtime_env/conda_utils.py @@ -161,7 +161,7 @@ def delete_conda_env(prefix: str, logger: Optional[logging.Logger] = None) -> bo def get_conda_env_list() -> list: """ - Get conda env list. + Get conda env list in full paths. """ conda_path = get_conda_bin_executable("conda") try: @@ -173,6 +173,33 @@ def get_conda_env_list() -> list: return envs +def get_conda_info_json() -> dict: + """ + Get `conda info --json` output. + """ + conda_path = get_conda_bin_executable("conda") + try: + exec_cmd([conda_path, "--help"], throw_on_error=False) + except EnvironmentError: + raise ValueError(f"Could not find Conda executable at {conda_path}.") + _, stdout, _ = exec_cmd([conda_path, "info", "--json"]) + return json.loads(stdout) + + +def get_conda_envs(conda_info: dict) -> List[Tuple[str, str]]: + """ + Gets the conda environments, as a list of (name, path) tuples. + """ + prefix = conda_info["conda_prefix"] + ret = [] + for env in conda_info["envs"]: + if env == prefix: + ret.append(("base", env)) + else: + ret.append((os.path.basename(env), env)) + return ret + + class ShellCommandException(Exception): pass diff --git a/python/ray/tests/test_runtime_env_complicated.py b/python/ray/tests/test_runtime_env_complicated.py index 0122a06dbc14..a71de6cf4a70 100644 --- a/python/ray/tests/test_runtime_env_complicated.py +++ b/python/ray/tests/test_runtime_env_complicated.py @@ -19,7 +19,11 @@ _current_py_version, ) -from ray._private.runtime_env.conda_utils import get_conda_env_list +from ray._private.runtime_env.conda_utils import ( + get_conda_env_list, + get_conda_info_json, + get_conda_envs, +) from ray._private.test_utils import ( run_string_as_driver, run_string_as_driver_nonblocking, @@ -213,6 +217,79 @@ def wrapped_version(self): assert ray.get(actor.wrapped_version.remote()) == package_version +@pytest.mark.skipif( + os.environ.get("CONDA_DEFAULT_ENV") is None, + reason="must be run from within a conda environment", +) +def test_base_full_path(conda_envs, shutdown_only): + """ + Test that `base` and its absolute path prefix can both work. + """ + ray.init() + + conda_info = get_conda_info_json() + prefix = conda_info["conda_prefix"] + + test_conda_envs = ["base", prefix] + + @ray.remote + def get_conda_env_name(): + return os.environ.get("CONDA_DEFAULT_ENV") + + # Basic conda runtime env + for conda_env in test_conda_envs: + runtime_env = {"conda": conda_env} + + task = get_conda_env_name.options(runtime_env=runtime_env) + assert ray.get(task.remote()) == "base" + + +@pytest.mark.skipif( + os.environ.get("CONDA_DEFAULT_ENV") is None, + reason="must be run from within a conda environment", +) +def test_task_actor_conda_env_full_path(conda_envs, shutdown_only): + ray.init() + + conda_info = get_conda_info_json() + prefix = conda_info["conda_prefix"] + + test_conda_envs = { + package_version: f"{prefix}/envs/package-{package_version}" + for package_version in EMOJI_VERSIONS + } + + # Basic conda runtime env + for package_version, conda_full_path in test_conda_envs.items(): + runtime_env = {"conda": conda_full_path} + print(f"Testing {package_version}, runtime env: {runtime_env}") + + task = get_emoji_version.options(runtime_env=runtime_env) + assert ray.get(task.remote()) == package_version + + actor = VersionActor.options(runtime_env=runtime_env).remote() + assert ray.get(actor.get_emoji_version.remote()) == package_version + + # Runtime env should inherit to nested task + @ray.remote + def wrapped_version(): + return ray.get(get_emoji_version.remote()) + + @ray.remote + class Wrapper: + def wrapped_version(self): + return ray.get(get_emoji_version.remote()) + + for package_version, conda_full_path in test_conda_envs.items(): + runtime_env = {"conda": conda_full_path} + + task = wrapped_version.options(runtime_env=runtime_env) + assert ray.get(task.remote()) == package_version + + actor = Wrapper.options(runtime_env=runtime_env).remote() + assert ray.get(actor.wrapped_version.remote()) == package_version + + @pytest.mark.skipif( os.environ.get("CONDA_DEFAULT_ENV") is None, reason="must be run from within a conda environment", @@ -329,6 +406,22 @@ def test_get_conda_env_dir(tmp_path): assert env_dir == str(tmp_path / "envs" / "tf2") +@pytest.mark.skipif( + os.environ.get("CONDA_DEFAULT_ENV") is None, + reason="must be run from within a conda environment", +) +def test_get_conda_envs(conda_envs): + """ + Tests that we can at least find 3 conda envs: base, and two envs we created. + """ + conda_info = get_conda_info_json() + envs = get_conda_envs(conda_info) + prefix = conda_info["conda_prefix"] + assert ("base", prefix) in envs + assert ("package-2.1.0", prefix + "/envs/package-2.1.0") in envs + assert ("package-2.2.0", prefix + "/envs/package-2.2.0") in envs + + @pytest.mark.skipif( os.environ.get("CONDA_EXE") is None, reason="Requires properly set-up conda shell", From e1294c61aab028d54fad9526f6a08fbfdd9c3c73 Mon Sep 17 00:00:00 2001 From: Ruiyang Wang Date: Tue, 28 May 2024 13:42:31 -0700 Subject: [PATCH 2/5] update rst Signed-off-by: Ruiyang Wang --- doc/source/ray-core/handling-dependencies.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/ray-core/handling-dependencies.rst b/doc/source/ray-core/handling-dependencies.rst index 69094903a5f8..7040e15f3026 100644 --- a/doc/source/ray-core/handling-dependencies.rst +++ b/doc/source/ray-core/handling-dependencies.rst @@ -402,12 +402,12 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime - Example: ``{"packages":["tensorflow", "requests"], "pip_check": False, "pip_version": "==22.0.2;python_version=='3.8.11'"}`` - When specifying a path to a ``requirements.txt`` file, the file must be present on your local machine and it must be a valid absolute path or relative filepath relative to your local current working directory, *not* relative to the `working_dir` specified in the `runtime_env`. - Furthermore, referencing local files *within* a `requirements.txt` file isn't directly supported (e.g., ``-r ./my-laptop/more-requirements.txt``, ``./my-pkg.whl``). Instead, use the `${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}` environment variable in the creation process. For example, use `-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-laptop/more-requirements.txt` or `${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-pkg.whl` to reference local files, while ensuring they're in the `working_dir`. + When specifying a path to a ``requirements.txt`` file, the file must be present on your local machine and it must be a valid absolute path or relative filepath relative to your local current working directory, *not* relative to the ``working_dir`` specified in the ``runtime_env``. + Furthermore, referencing local files *within* a ``requirements.txt`` file isn't directly supported (e.g., ``-r ./my-laptop/more-requirements.txt``, ``./my-pkg.whl``). Instead, use the ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}`` environment variable in the creation process. For example, use ``-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-laptop/more-requirements.txt`` or ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-pkg.whl`` to reference local files, while ensuring they're in the ``working_dir``. - ``conda`` (dict | str): Either (1) a dict representing the conda environment YAML, (2) a string containing the path to a local `conda “environment.yml” `_ file, - or (3) the name of a local conda environment already installed on each node in your cluster (e.g., ``"pytorch_p36"``). + or (3) the name of a local conda environment already installed on each node in your cluster (e.g., ``"pytorch_p36"``) or its absolute path (e.g. ``"/home/youruser/anaconda3/envs/i37197"``) . In the first two cases, the Ray and Python dependencies will be automatically injected into the environment to ensure compatibility, so there is no need to manually include them. The Python and Ray version must match that of the cluster, so you likely should not specify them manually. Note that the ``conda`` and ``pip`` keys of ``runtime_env`` cannot both be specified at the same time---to use them together, please use ``conda`` and add your pip dependencies in the ``"pip"`` field in your conda ``environment.yaml``. @@ -418,8 +418,8 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime - Example: ``"pytorch_p36"`` - When specifying a path to a ``environment.yml`` file, the file must be present on your local machine and it must be a valid absolute path or a relative filepath relative to your local current working directory, *not* relative to the `working_dir` specified in the `runtime_env`. - Furthermore, referencing local files *within* a `environment.yml` file isn't directly supported (e.g., ``-r ./my-laptop/more-requirements.txt``, ``./my-pkg.whl``). Instead, use the `${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}` environment variable in the creation process. For example, use `-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-laptop/more-requirements.txt` or `${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-pkg.whl` to reference local files, while ensuring they're in the `working_dir`. + When specifying a path to a ``environment.yml`` file, the file must be present on your local machine and it must be a valid absolute path or a relative filepath relative to your local current working directory, *not* relative to the ``working_dir`` specified in the ``runtime_env``. + Furthermore, referencing local files *within* a ``environment.yml`` file isn't directly supported (e.g., ``-r ./my-laptop/more-requirements.txt``, ``./my-pkg.whl``). Instead, use the ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}`` environment variable in the creation process. For example, use ``-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-laptop/more-requirements.txt`` or ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-pkg.whl`` to reference local files, while ensuring they're in the ``working_dir``. - ``env_vars`` (Dict[str, str]): Environment variables to set. Environment variables already set on the cluster will still be visible to the Ray workers; so there is no need to include ``os.environ`` or similar in the ``env_vars`` field. From ac913f91e79f9066958ee6653a9f704f31f9e2a9 Mon Sep 17 00:00:00 2001 From: Ruiyang Wang Date: Tue, 28 May 2024 13:43:23 -0700 Subject: [PATCH 3/5] fix rst Signed-off-by: Ruiyang Wang --- doc/source/ray-core/handling-dependencies.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/ray-core/handling-dependencies.rst b/doc/source/ray-core/handling-dependencies.rst index 7040e15f3026..bd223e2a4772 100644 --- a/doc/source/ray-core/handling-dependencies.rst +++ b/doc/source/ray-core/handling-dependencies.rst @@ -407,7 +407,7 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime - ``conda`` (dict | str): Either (1) a dict representing the conda environment YAML, (2) a string containing the path to a local `conda “environment.yml” `_ file, - or (3) the name of a local conda environment already installed on each node in your cluster (e.g., ``"pytorch_p36"``) or its absolute path (e.g. ``"/home/youruser/anaconda3/envs/i37197"``) . + or (3) the name of a local conda environment already installed on each node in your cluster (e.g., ``"pytorch_p36"``) or its absolute path (e.g. ``"/home/youruser/anaconda3/envs/pytorch_p36"``) . In the first two cases, the Ray and Python dependencies will be automatically injected into the environment to ensure compatibility, so there is no need to manually include them. The Python and Ray version must match that of the cluster, so you likely should not specify them manually. Note that the ``conda`` and ``pip`` keys of ``runtime_env`` cannot both be specified at the same time---to use them together, please use ``conda`` and add your pip dependencies in the ``"pip"`` field in your conda ``environment.yaml``. @@ -418,6 +418,8 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime - Example: ``"pytorch_p36"`` + - Example: ``"/home/youruser/anaconda3/envs/pytorch_p36"`` + When specifying a path to a ``environment.yml`` file, the file must be present on your local machine and it must be a valid absolute path or a relative filepath relative to your local current working directory, *not* relative to the ``working_dir`` specified in the ``runtime_env``. Furthermore, referencing local files *within* a ``environment.yml`` file isn't directly supported (e.g., ``-r ./my-laptop/more-requirements.txt``, ``./my-pkg.whl``). Instead, use the ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}`` environment variable in the creation process. For example, use ``-r ${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-laptop/more-requirements.txt`` or ``${RAY_RUNTIME_ENV_CREATE_WORKING_DIR}/my-pkg.whl`` to reference local files, while ensuring they're in the ``working_dir``. From 8996e64926bebc84f93e031a3efa1c007785fd11 Mon Sep 17 00:00:00 2001 From: Ruiyang Wang Date: Tue, 28 May 2024 21:28:33 -0700 Subject: [PATCH 4/5] add comment about the dict Signed-off-by: Ruiyang Wang --- python/ray/_private/runtime_env/conda_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/ray/_private/runtime_env/conda_utils.py b/python/ray/_private/runtime_env/conda_utils.py index a33916c5e46f..b3ca15300224 100644 --- a/python/ray/_private/runtime_env/conda_utils.py +++ b/python/ray/_private/runtime_env/conda_utils.py @@ -176,6 +176,14 @@ def get_conda_env_list() -> list: def get_conda_info_json() -> dict: """ Get `conda info --json` output. + + Returns dict of conda info. See [1] for more details. We mostly care about these + keys: + + - `conda_prefix`: str The path to the conda installation. + - `envs`: List[str] absolute paths to conda environments. + + [1] https://github.com/conda/conda/blob/main/conda/cli/main_info.py """ conda_path = get_conda_bin_executable("conda") try: From c0835a2938308aeda49beba79a7ac39103d284a0 Mon Sep 17 00:00:00 2001 From: Ruiyang Wang Date: Tue, 16 Jul 2024 09:56:18 -0700 Subject: [PATCH 5/5] fix test Signed-off-by: Ruiyang Wang --- python/ray/tests/test_runtime_env_conda_and_pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tests/test_runtime_env_conda_and_pip.py b/python/ray/tests/test_runtime_env_conda_and_pip.py index 117b568dc6d5..d241e8c0a90d 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip.py @@ -225,7 +225,7 @@ def f(): for ref in refs: with pytest.raises(ray.exceptions.RuntimeEnvSetupError) as exc_info: ray.get(ref) - assert "doesn't exist from the output of `conda env list --json`" in str( + assert "doesn't exist from the output of `conda info --json`" in str( exc_info.value ) # noqa