Skip to content

Commit

Permalink
uv supports requirement file
Browse files Browse the repository at this point in the history
Signed-off-by: dentiny <dentinyhao@gmail.com>
  • Loading branch information
dentiny committed Nov 7, 2024
1 parent a49978f commit 230d0a2
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 19 deletions.
32 changes: 18 additions & 14 deletions python/ray/_private/runtime_env/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ def validate_uri(uri: str):
raise ValueError("Only .zip or .whl files supported for remote URIs.")


def _handle_local_deps_requirement_file(requirements_file: str):
"""Read the given [requirements_file], and return all required dependencies."""
requirements_path = Path(requirements_file)
if not requirements_path.is_file():
raise ValueError(f"{requirements_path} is not a valid file")
return requirements_path.read_text().strip().split("\n")


def parse_and_validate_py_modules(py_modules: List[str]) -> List[str]:
"""Parses and validates a 'py_modules' option.
Expand Down Expand Up @@ -108,10 +116,8 @@ def parse_and_validate_conda(conda: Union[str, dict]) -> Union[str, dict]:


# TODO(hjiang): More package installation options to implement:
# 1. Allow users to pass in a local requirements.txt file, which relates to all
# packages to install;
# 2. Allow specific version of `uv` to use; as of now we only use default version.
# 3. `pip_check` has different semantics for `uv` and `pip`, see
# 1. Allow specific version of `uv` to use; as of now we only use default version.
# 2. `pip_check` has different semantics for `uv` and `pip`, see
# https://github.com/astral-sh/uv/pull/2544/files, consider whether we need to support
# it; or simply ignore the field when people come from `pip`.
def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]:
Expand All @@ -120,7 +126,8 @@ def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]:
The value of the input 'uv' field can be one of two cases:
1) A List[str] describing the requirements. This is passed through.
Example usage: ["tensorflow", "requests"]
2) A python dictionary that has one field:
2) a string containing the path to a local pip “requirements.txt” file.
3) A python dictionary that has one field:
a) packages (required, List[str]): a list of uv packages, it same as 1).
The returned parsed value will be a list of packages. If a Ray library
Expand All @@ -136,7 +143,10 @@ def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]:
)

result: str = ""
if isinstance(uv, list) and all(isinstance(dep, str) for dep in uv):
if isinstance(uv, str):
uv_list = _handle_local_deps_requirement_file(uv)
result = dict(packages=uv_list)
elif isinstance(uv, list) and all(isinstance(dep, str) for dep in uv):
result = dict(packages=uv)
elif isinstance(uv, dict):
if set(uv.keys()) - {"packages"}:
Expand Down Expand Up @@ -192,12 +202,6 @@ def parse_and_validate_pip(pip: Union[str, List[str], Dict]) -> Optional[Dict]:
"""
assert pip is not None

def _handle_local_pip_requirement_file(pip_file: str):
pip_path = Path(pip_file)
if not pip_path.is_file():
raise ValueError(f"{pip_path} is not a valid file")
return pip_path.read_text().strip().split("\n")

result = None
if sys.platform == "win32":
logger.warning(
Expand All @@ -207,7 +211,7 @@ def _handle_local_pip_requirement_file(pip_file: str):
)
if isinstance(pip, str):
# We have been given a path to a requirements.txt file.
pip_list = _handle_local_pip_requirement_file(pip)
pip_list = _handle_local_deps_requirement_file(pip)
result = dict(packages=pip_list, pip_check=False)
elif isinstance(pip, list) and all(isinstance(dep, str) for dep in pip):
result = dict(packages=pip, pip_check=False)
Expand Down Expand Up @@ -237,7 +241,7 @@ def _handle_local_pip_requirement_file(pip_file: str):
f"runtime_env['pip'] must include field 'packages', but got {pip}"
)
elif isinstance(pip["packages"], str):
result["packages"] = _handle_local_pip_requirement_file(pip["packages"])
result["packages"] = _handle_local_deps_requirement_file(pip["packages"])
elif not isinstance(pip["packages"], list):
raise ValueError(
"runtime_env['pip']['packages'] must be of type str of list, "
Expand Down
1 change: 1 addition & 0 deletions python/ray/tests/unit/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ py_test(
name = "test_runtime_env_uv",
srcs = ["test_runtime_env_uv.py"],
tags = ["team:core"],
data = ["test_requirements.txt"],
deps = [
"//python/ray/_private/runtime_env:uv",
],
Expand Down
12 changes: 7 additions & 5 deletions python/ray/tests/unit/test_runtime_env_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def test_directory():


class TestVaidationUv:
def test_parse_and_validate_uv(self):
def test_parse_and_validate_uv(self, test_directory):
# Valid case w/o duplication.
result = validation.parse_and_validate_uv({"packages": ["tensorflow"]})
assert result == {"packages": ["tensorflow"]}
Expand All @@ -46,14 +46,16 @@ def test_parse_and_validate_uv(self):
)
assert result == {"packages": ["requests==1.0.0", "aiohttp", "ray[serve]"]}

# Invalid case, `str` is not supported for now.
with pytest.raises(TypeError):
result = validation.parse_and_validate_uv("./requirements.txt")

# Invalid case, unsupport keys.
with pytest.raises(ValueError):
result = validation.parse_and_validate_uv({"random_key": "random_value"})

# Valid requirement files.
_, requirements_file = test_directory
requirements_file = requirements_file.resolve()
result = validation.parse_and_validate_uv(str(requirements_file))
assert result == {"packages": ["requests==1.0.0", "pip-install-test"]}


class TestValidatePip:
def test_validate_pip_invalid_types(self):
Expand Down

0 comments on commit 230d0a2

Please sign in to comment.