[deps] adding include_setuptools flag for depset config (#58580)

Adding optional `include_setuptools` flag for depset configuration

If the flag is set on a depset config --unsafe-package setuptools will
not be included for depset compilation

If the flag does not exist (default false) on a depset config
--unsafe-package setuptools will be appended to the default arguments

---------

Signed-off-by: elliot-barn <elliot.barnwell@anyscale.com>
Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com>

Author: elliot-barn

ci/raydepsets/cli.py

@@ -17,7 +17,6 @@
 
 DEFAULT_UV_FLAGS = """
     --generate-hashes
-    --unsafe-package setuptools
     --index-url https://pypi.org/simple
     --index-strategy unsafe-best-match
     --no-strip-markers
@@ -275,6 +274,7 @@ def execute_depset(self, depset: Depset):
                 append_flags=depset.append_flags,
                 override_flags=depset.override_flags,
                 packages=depset.packages,
+                include_setuptools=depset.include_setuptools,
             )
         elif depset.operation == "subset":
             self.subset(
@@ -284,6 +284,7 @@ def execute_depset(self, depset: Depset):
                 override_flags=depset.override_flags,
                 name=depset.name,
                 output=depset.output,
+                include_setuptools=depset.include_setuptools,
             )
         elif depset.operation == "expand":
             self.expand(
@@ -294,6 +295,7 @@ def execute_depset(self, depset: Depset):
                 override_flags=depset.override_flags,
                 name=depset.name,
                 output=depset.output,
+                include_setuptools=depset.include_setuptools,
             )
         click.echo(f"Dependency set {depset.name} compiled successfully")
 
@@ -306,10 +308,13 @@ def compile(
         override_flags: Optional[List[str]] = None,
         packages: Optional[List[str]] = None,
         requirements: Optional[List[str]] = None,
+        include_setuptools: Optional[bool] = False,
     ):
         """Compile a dependency set."""
         args = DEFAULT_UV_FLAGS.copy()
         stdin = None
+        if not include_setuptools:
+            args.extend(_flatten_flags(["--unsafe-package setuptools"]))
         if self._uv_cache_dir:
             args.extend(["--cache-dir", self._uv_cache_dir])
         if override_flags:
@@ -338,6 +343,7 @@ def subset(
         output: str = None,
         append_flags: Optional[List[str]] = None,
         override_flags: Optional[List[str]] = None,
+        include_setuptools: Optional[bool] = False,
     ):
         """Subset a dependency set."""
         source_depset = _get_depset(self.config.depsets, source_depset)
@@ -349,6 +355,7 @@ def subset(
             output=output,
             append_flags=append_flags,
             override_flags=override_flags,
+            include_setuptools=include_setuptools,
         )
 
     def expand(
@@ -360,6 +367,7 @@ def expand(
         output: str = None,
         append_flags: Optional[List[str]] = None,
         override_flags: Optional[List[str]] = None,
+        include_setuptools: Optional[bool] = False,
     ):
         """Expand a dependency set."""
         # handle both depsets and requirements
@@ -377,6 +385,7 @@ def expand(
             output=output,
             append_flags=append_flags,
             override_flags=override_flags,
+            include_setuptools=include_setuptools,
         )
 
     def read_lock_file(self, file_path: Path) -> List[str]:

ci/raydepsets/tests/test_cli.py

@@ -319,6 +319,36 @@ def test_append_uv_flags_with_space_in_flag(self):
             output_text = output_file.read_text()
             assert "--python-version 3.10" in output_text
 
+    def test_include_setuptools(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            copy_data_to_tmpdir(tmpdir)
+            manager = _create_test_manager(tmpdir)
+            manager.compile(
+                constraints=[],
+                requirements=["requirements_test.txt"],
+                name="general_depset",
+                output="requirements_compiled_general.txt",
+                include_setuptools=True,
+            )
+            output_file = Path(tmpdir) / "requirements_compiled_general.txt"
+            output_text = output_file.read_text()
+            assert "--unsafe-package setuptools" not in output_text
+
+    def test_ignore_setuptools(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            copy_data_to_tmpdir(tmpdir)
+            manager = _create_test_manager(tmpdir)
+            manager.compile(
+                constraints=[],
+                requirements=["requirements_test.txt"],
+                name="general_depset",
+                output="requirements_compiled_general.txt",
+                include_setuptools=False,
+            )
+            output_file = Path(tmpdir) / "requirements_compiled_general.txt"
+            output_text = output_file.read_text()
+            assert "--unsafe-package setuptools" in output_text
+
     def test_override_uv_flag_single_flag(self):
         expected_flags = DEFAULT_UV_FLAGS.copy()
         expected_flags.remove("--index-strategy")
@@ -334,12 +364,12 @@ def test_override_uv_flag_single_flag(self):
 
     def test_override_uv_flag_multiple_flags(self):
         expected_flags = DEFAULT_UV_FLAGS.copy()
-        expected_flags.remove("--unsafe-package")
-        expected_flags.remove("setuptools")
-        expected_flags.extend(["--unsafe-package", "dummy"])
+        expected_flags.remove("--index-url")
+        expected_flags.remove("https://pypi.org/simple")
+        expected_flags.extend(["--index-url", "https://dummyurl.com"])
         assert (
             _override_uv_flags(
-                ["--unsafe-package dummy"],
+                ["--index-url https://dummyurl.com"],
                 DEFAULT_UV_FLAGS.copy(),
             )
             == expected_flags

ci/raydepsets/tests/test_data/test.depsets.yaml

@@ -60,3 +60,4 @@ depsets:
     constraints:
       - requirement_constraints_expand.txt
     output: requirements_compiled_nested_expand.txt
+    include_setuptools: true

ci/raydepsets/workspace.py

@@ -25,6 +25,7 @@ class Depset:
     source_depset: Optional[str] = None
     depsets: Optional[List[str]] = None
     pre_hooks: Optional[List[str]] = None
+    include_setuptools: Optional[bool] = False
 
 
 def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet):
@@ -54,6 +55,7 @@ def _dict_to_depset(depset: dict, config_name: str) -> Depset:
         append_flags=depset.get("append_flags", []),
         pre_hooks=depset.get("pre_hooks", []),
         packages=depset.get("packages", []),
+        include_setuptools=depset.get("include_setuptools", False),
         config_name=config_name,
     )
 

python/deplocks/base_extra/ray_base_extra_py3.10.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.10 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.10.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.10.lock
+#    uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.10 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.10.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.10.lock
 --index-url https://pypi.org/simple
 
 adlfs==2023.8.0 \

python/deplocks/base_extra/ray_base_extra_py3.11.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.11 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.11.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.11.lock
+#    uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.11 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.11.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.11.lock
 --index-url https://pypi.org/simple
 
 adlfs==2023.8.0 \

python/deplocks/base_extra/ray_base_extra_py3.12.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.12 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.12.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.12.lock
+#    uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.12 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.12.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.12.lock
 --index-url https://pypi.org/simple
 
 adlfs==2023.8.0 \

python/deplocks/base_extra/ray_base_extra_py3.9.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package ray --python-version=3.9 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.9.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.9.lock
+#    uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --unsafe-package ray --python-version=3.9 --python-platform=linux -c release/ray_release/byod/ray_base_extra_testdeps_py3.9.lock docker/base-deps/requirements.in docker/base-extra/requirements.in -o python/deplocks/base_extra/ray_base_extra_py3.9.lock
 --index-url https://pypi.org/simple
 
 adlfs==2023.8.0 \

python/deplocks/docs/docbuild_depset_py3.10.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.10 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.10.lock
+#    uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.10 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.10.lock
 --index-url https://pypi.org/simple
 
 accessible-pygments==0.0.5 \

python/deplocks/docs/docbuild_depset_py3.12.lock

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --generate-hashes --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.12 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.12.lock
+#    uv pip compile --generate-hashes --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --unsafe-package setuptools --python-version=3.12 --python-platform=linux --unsafe-package ray doc/requirements-doc.txt -o python/deplocks/docs/docbuild_depset_py3.12.lock
 --index-url https://pypi.org/simple
 
 accessible-pygments==0.0.5 \