Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion ci/raydepsets/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

DEFAULT_UV_FLAGS = """
--generate-hashes
--unsafe-package setuptools
--index-url https://pypi.org/simple
--index-strategy unsafe-best-match
--no-strip-markers
Expand Down Expand Up @@ -275,6 +274,7 @@ def execute_depset(self, depset: Depset):
append_flags=depset.append_flags,
override_flags=depset.override_flags,
packages=depset.packages,
include_setuptools=depset.include_setuptools,
)
elif depset.operation == "subset":
self.subset(
Expand All @@ -284,6 +284,7 @@ def execute_depset(self, depset: Depset):
override_flags=depset.override_flags,
name=depset.name,
output=depset.output,
include_setuptools=depset.include_setuptools,
)
elif depset.operation == "expand":
self.expand(
Expand All @@ -294,6 +295,7 @@ def execute_depset(self, depset: Depset):
override_flags=depset.override_flags,
name=depset.name,
output=depset.output,
include_setuptools=depset.include_setuptools,
)
click.echo(f"Dependency set {depset.name} compiled successfully")

Expand All @@ -306,10 +308,13 @@ def compile(
override_flags: Optional[List[str]] = None,
packages: Optional[List[str]] = None,
requirements: Optional[List[str]] = None,
include_setuptools: Optional[bool] = False,
):
"""Compile a dependency set."""
args = DEFAULT_UV_FLAGS.copy()
stdin = None
if not include_setuptools:
args.extend(_flatten_flags(["--unsafe-package setuptools"]))
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: Override Logic Ignores Setuptools Flag

The include_setuptools flag is applied before override_flags, causing it to be ignored when override flags contain --unsafe-package. The _override_uv_flags function removes all instances of flag names that appear in the override list, so if override flags include --unsafe-package ray, it removes the previously added --unsafe-package setuptools. The setuptools flag should be added after applying override flags to ensure the include_setuptools setting is respected.

Fix in Cursor Fix in Web

if self._uv_cache_dir:
args.extend(["--cache-dir", self._uv_cache_dir])
if override_flags:
Expand Down Expand Up @@ -338,6 +343,7 @@ def subset(
output: str = None,
append_flags: Optional[List[str]] = None,
override_flags: Optional[List[str]] = None,
include_setuptools: Optional[bool] = False,
):
"""Subset a dependency set."""
source_depset = _get_depset(self.config.depsets, source_depset)
Expand All @@ -349,6 +355,7 @@ def subset(
output=output,
append_flags=append_flags,
override_flags=override_flags,
include_setuptools=include_setuptools,
)

def expand(
Expand All @@ -360,6 +367,7 @@ def expand(
output: str = None,
append_flags: Optional[List[str]] = None,
override_flags: Optional[List[str]] = None,
include_setuptools: Optional[bool] = False,
):
"""Expand a dependency set."""
# handle both depsets and requirements
Expand All @@ -377,6 +385,7 @@ def expand(
output=output,
append_flags=append_flags,
override_flags=override_flags,
include_setuptools=include_setuptools,
)

def read_lock_file(self, file_path: Path) -> List[str]:
Expand Down
38 changes: 34 additions & 4 deletions ci/raydepsets/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,36 @@ def test_append_uv_flags_with_space_in_flag(self):
output_text = output_file.read_text()
assert "--python-version 3.10" in output_text

def test_include_setuptools(self):
with tempfile.TemporaryDirectory() as tmpdir:
copy_data_to_tmpdir(tmpdir)
manager = _create_test_manager(tmpdir)
manager.compile(
constraints=[],
requirements=["requirements_test.txt"],
name="general_depset",
output="requirements_compiled_general.txt",
include_setuptools=True,
)
output_file = Path(tmpdir) / "requirements_compiled_general.txt"
output_text = output_file.read_text()
assert "--unsafe-package setuptools" not in output_text

def test_ignore_setuptools(self):
with tempfile.TemporaryDirectory() as tmpdir:
copy_data_to_tmpdir(tmpdir)
manager = _create_test_manager(tmpdir)
manager.compile(
constraints=[],
requirements=["requirements_test.txt"],
name="general_depset",
output="requirements_compiled_general.txt",
include_setuptools=False,
)
output_file = Path(tmpdir) / "requirements_compiled_general.txt"
output_text = output_file.read_text()
assert "--unsafe-package setuptools" in output_text

Comment on lines +322 to +351
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The tests test_include_setuptools and test_ignore_setuptools are very similar and can be combined into a single, parameterized test using pytest.mark.parametrize. This will reduce code duplication and make the tests more maintainable.

You'll need to ensure pytest is imported in this file.

    @pytest.mark.parametrize(
        "include_setuptools, expect_unsafe_flag",
        [
            (True, False),
            (False, True),
        ],
    )
    def test_setuptools_handling(self, include_setuptools, expect_unsafe_flag):
        with tempfile.TemporaryDirectory() as tmpdir:
            copy_data_to_tmpdir(tmpdir)
            manager = _create_test_manager(tmpdir)
            manager.compile(
                constraints=[],
                requirements=["requirements_test.txt"],
                name="general_depset",
                output="requirements_compiled_general.txt",
                include_setuptools=include_setuptools,
            )
            output_file = Path(tmpdir) / "requirements_compiled_general.txt"
            output_text = output_file.read_text()
            has_unsafe_flag = "--unsafe-package setuptools" in output_text
            assert has_unsafe_flag is expect_unsafe_flag

def test_override_uv_flag_single_flag(self):
expected_flags = DEFAULT_UV_FLAGS.copy()
expected_flags.remove("--index-strategy")
Expand All @@ -334,12 +364,12 @@ def test_override_uv_flag_single_flag(self):

def test_override_uv_flag_multiple_flags(self):
expected_flags = DEFAULT_UV_FLAGS.copy()
expected_flags.remove("--unsafe-package")
expected_flags.remove("setuptools")
expected_flags.extend(["--unsafe-package", "dummy"])
expected_flags.remove("--index-url")
expected_flags.remove("https://pypi.org/simple")
expected_flags.extend(["--index-url", "https://dummyurl.com"])
assert (
_override_uv_flags(
["--unsafe-package dummy"],
["--index-url https://dummyurl.com"],
DEFAULT_UV_FLAGS.copy(),
)
== expected_flags
Expand Down
1 change: 1 addition & 0 deletions ci/raydepsets/tests/test_data/test.depsets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,4 @@ depsets:
constraints:
- requirement_constraints_expand.txt
output: requirements_compiled_nested_expand.txt
include_setuptools: true
2 changes: 2 additions & 0 deletions ci/raydepsets/workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class Depset:
source_depset: Optional[str] = None
depsets: Optional[List[str]] = None
pre_hooks: Optional[List[str]] = None
include_setuptools: Optional[bool] = False


def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet):
Expand Down Expand Up @@ -54,6 +55,7 @@ def _dict_to_depset(depset: dict, config_name: str) -> Depset:
append_flags=depset.get("append_flags", []),
pre_hooks=depset.get("pre_hooks", []),
packages=depset.get("packages", []),
include_setuptools=depset.get("include_setuptools", False),
config_name=config_name,
)

Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/base_extra/ray_base_extra_py3.10.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.10 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.10.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.10.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.10 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.10.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.10.lock
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm... I thought these flags are sorted already?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no i reverted the flag sorting. Order matters and didnt want to sort lexigraphically. Will handle that with the flag interface

--index-url https://pypi.org/simple

adlfs==2023.8.0 \
Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/base_extra/ray_base_extra_py3.11.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.11 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.11.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.11.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.11 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.11.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.11.lock
--index-url https://pypi.org/simple

adlfs==2023.8.0 \
Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/base_extra/ray_base_extra_py3.12.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.12 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.12.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.12.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.12 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.12.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.12.lock
--index-url https://pypi.org/simple

adlfs==2023.8.0 \
Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/base_extra/ray_base_extra_py3.9.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.9 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.9.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.9.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.9 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.9.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.9.lock
--index-url https://pypi.org/simple

adlfs==2023.8.0 \
Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/docs/docbuild_depset_py3.10.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.10 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.10.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.10 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.10.lock
--index-url https://pypi.org/simple

accessible-pygments==0.0.5 \
Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/docs/docbuild_depset_py3.12.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.12 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.12.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.12 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.12.lock
--index-url https://pypi.org/simple

accessible-pygments==0.0.5 \
Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/docs/docbuild_depset_py3.9.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.9 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.9.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.9 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.9.lock
--index-url https://pypi.org/simple

accessible-pygments==0.0.5 \
Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/llm/ray_py311_cpu.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cpu -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cpu.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cpu -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cpu.lock
--index-url https://pypi.org/simple
--extra-index-url https://download.pytorch.org/whl/cpu

Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/llm/ray_py311_cu128.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cu128 -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu128.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cu128 -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu128.lock
--index-url https://pypi.org/simple
--extra-index-url https://download.pytorch.org/whl/cu128

Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/llm/ray_test_py311_cpu.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cpu -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/base-test-requirements.txt python/requirements/cloud-requirements.txt -o python/deplocks/llm/ray_test_py311_cpu.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cpu -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/base-test-requirements.txt python/requirements/cloud-requirements.txt -o python/deplocks/llm/ray_test_py311_cpu.lock
--index-url https://pypi.org/simple
--extra-index-url https://download.pytorch.org/whl/cpu

Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/llm/ray_test_py311_cu128.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cu128 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/base-test-requirements.txt python/requirements/cloud-requirements.txt -o python/deplocks/llm/ray_test_py311_cu128.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cu128 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/base-test-requirements.txt python/requirements/cloud-requirements.txt -o python/deplocks/llm/ray_test_py311_cu128.lock
--index-url https://pypi.org/simple
--extra-index-url https://download.pytorch.org/whl/cu128

Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/llm/rayllm_py311_cpu.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cpu -c python/deplocks/llm/rayllm_test_py311_cpu.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cpu.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cpu -c python/deplocks/llm/rayllm_test_py311_cpu.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cpu.lock
--index-url https://pypi.org/simple
--extra-index-url https://download.pytorch.org/whl/cpu

Expand Down
2 changes: 1 addition & 1 deletion python/deplocks/llm/rayllm_py311_cu128.lock
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
# uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cu128 -c python/deplocks/llm/rayllm_test_py311_cu128.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu128.lock
# uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.11 --unsafe-package ray --python-platform=linux --extra-index-url https://download.pytorch.org/whl/cu128 -c python/deplocks/llm/rayllm_test_py311_cu128.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu128.lock
--index-url https://pypi.org/simple
--extra-index-url https://download.pytorch.org/whl/cu128

Expand Down
Loading