Merge conflicts

Signed-off-by: joshlee <joshlee@anyscale.com>

Author: Sparks0219

.buildkite/core.rayci.yml

@@ -62,7 +62,7 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/_common/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --except-tags custom_setup
+        --except-tags custom_setup,cgroup
         --install-mask all-ray-libraries
 
   - label: ":ray: core: python {{matrix.python}} tests ({{matrix.worker_id}})"
@@ -76,7 +76,7 @@ steps:
         --install-mask all-ray-libraries
         --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
         --python-version {{matrix.python}}
-        --except-tags custom_setup
+        --except-tags custom_setup,cgroup
     depends_on: corebuild-multipy
     matrix:
       setup:
@@ -105,7 +105,7 @@ steps:
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --test-env=TEST_EXTERNAL_REDIS=1
-        --except-tags custom_setup
+        --except-tags custom_setup,cgroup
 
   - label: ":ray: core: memory pressure tests"
     tags:
@@ -261,7 +261,7 @@ steps:
         --test-env=RAY_MINIMAL=1
         --test-env=EXPECTED_PYTHON_VERSION={{matrix}}
         --only-tags minimal
-        --except-tags basic_test,manual
+        --except-tags basic_test,manual,cgroup
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dashboard/... core
         --parallelism-per-worker 3
         --python-version {{matrix}}
@@ -366,7 +366,7 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/... //doc/... core
         --install-mask all-ray-libraries
         --run-flaky-tests
-        --except-tags multi_gpu
+        --except-tags multi_gpu,cgroup
     depends_on:
       - corebuild
 

.buildkite/data.rayci.yml

@@ -40,7 +40,7 @@ steps:
     tags:
       - data
     instance_type: medium
-    parallelism: 2
+    parallelism: 8
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
@@ -54,8 +54,11 @@ steps:
       - data
       - data_non_parallel
     instance_type: medium
+    parallelism: 3
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
+        --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --build-name data9build
         --only-tags data_non_parallel
     depends_on: data9build
@@ -65,7 +68,7 @@ steps:
       - python
       - data
     instance_type: medium
-    parallelism: 2
+    parallelism: 8
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
@@ -80,8 +83,11 @@ steps:
       - data
       - data_non_parallel
     instance_type: medium
+    parallelism: 3
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
+        --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --build-name datalbuild
         --only-tags data_non_parallel
     depends_on: datalbuild

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,25 +1,45 @@
-<!-- Thank you for your contribution! Please review https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before opening a pull request. -->
+<!-- Thank you for contributing to Ray! 🚀 -->
+<!-- Please review https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before opening a pull request. -->
+<!-- 💡 Tip: Mark as draft if you want early feedback, or ready for review when it's complete -->
 
-<!-- Please add a reviewer to the assignee section when you create a PR. If you don't have the access to it, we will shortly find a reviewer and assign them to your PR. -->
+## Description
 
-## Why are these changes needed?
+<!-- Briefly describe what this PR accomplishes and why it's needed -->
 
-<!-- Please give a short summary of the change and the problem this solves. -->
+## Related issues
 
-## Related issue number
+<!-- Link related issues: "Fixes #1234", "Closes #1234", or "Related to #1234" -->
 
-<!-- For example: "Closes #1234" -->
+## Types of change
 
-## Checks
+- [ ] Bug fix 🐛
+- [ ] New feature ✨
+- [ ] Enhancement 🚀
+- [ ] Code refactoring 🔧
+- [ ] Documentation update 📖
+- [ ] Chore 🧹
+- [ ] Style 🎨
 
-- [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR.
-- [ ] I've run pre-commit jobs to lint the changes in this PR. ([pre-commit setup](https://docs.ray.io/en/latest/ray-contribute/getting-involved.html#lint-and-formatting))
-- [ ] I've included any doc changes needed for https://docs.ray.io/en/master/.
-    - [ ] I've added any new APIs to the API Reference. For example, if I added a
-           method in Tune, I've added it in `doc/source/tune/api/` under the
-           corresponding `.rst` file.
-- [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/
-- Testing Strategy
-   - [ ] Unit tests
-   - [ ] Release tests
-   - [ ] This PR is not tested :(
+## Checklist
+
+**Does this PR introduce breaking changes?**
+- [ ] Yes ⚠️
+- [ ] No
+<!-- If yes, describe what breaks and how users should migrate -->
+
+**Testing:**
+- [ ] Added/updated tests for my changes
+- [ ] Tested the changes manually
+- [ ] This PR is not tested ❌ _(please explain why)_
+
+**Code Quality:**
+- [ ] Signed off every commit (`git commit -s`)
+- [ ] Ran pre-commit hooks ([setup guide](https://docs.ray.io/en/latest/ray-contribute/getting-involved.html#lint-and-formatting))
+
+**Documentation:**
+- [ ] Updated documentation (if applicable) ([contribution guide](https://docs.ray.io/en/latest/ray-contribute/docs.html))
+- [ ] Added new APIs to `doc/source/` (if applicable)
+
+## Additional context
+
+<!-- Optional: Add screenshots, examples, performance impact, breaking change details -->

ci/lint/pre-push

@@ -2,14 +2,14 @@
 
 echo "Linting changes as part of pre-push hook"
 echo ""
-echo "ci/lint/format.sh:"
-ci/lint/format.sh
+echo "pre-commit:"
+pre-commit run --from-ref master --to-ref HEAD
 
 lint_exit_status=$?
 if [ $lint_exit_status -ne 0 ]; then
 	echo ""
 	echo "Linting changes failed."
-	echo "Please make sure 'ci/lint/format.sh'"\
+	echo "Please make sure 'pre-commit'"\
 		"runs with no errors before pushing."
 	echo "If you want to ignore this and push anyways,"\
 		"re-run with '--no-verify'."

ci/lint/pydoclint-baseline.txt

@@ -1528,10 +1528,6 @@ python/ray/serve/_private/api.py
     DOC201: Function `serve_start` does not have a return section in docstring
 --------------------
 python/ray/serve/_private/application_state.py
-    DOC001: Method `__init__` Potential formatting errors in docstring. Error message: No specification for "Args": ""
-    DOC001: Function/method `__init__`: Potential formatting errors in docstring. Error message: No specification for "Args": "" (Note: DOC001 could trigger other unrelated violations under this function/method too. Please fix the docstring formatting first.)
-    DOC101: Method `ApplicationState.__init__`: Docstring contains fewer arguments than in function signature.
-    DOC103: Method `ApplicationState.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [deployment_state_manager: DeploymentStateManager, endpoint_state: EndpointState, logging_config: LoggingConfig, name: str].
     DOC103: Method `ApplicationStateManager.deploy_app`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [deployment_args: List[Dict]]. Arguments in the docstring but not in the function signature: [deployment_args_list: ].
     DOC102: Function `override_deployment_info`: Docstring contains more arguments than in function signature.
     DOC103: Function `override_deployment_info`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the docstring but not in the function signature: [app_name: ].

ci/pipeline/test_conditional_testing.py

@@ -16,7 +16,7 @@
 
 _TESTS_YAML = """
 ci/pipeline/test_conditional_testing.py: lint tools
-python/ray/data/__init__.py: lint data linux_wheels ml train
+python/ray/data/__init__.py: lint data ml train
 doc/index.md: lint
 
 python/ray/air/__init__.py: lint ml train tune data linux_wheels

ci/pipeline/test_rules.txt

@@ -39,7 +39,7 @@ ci/docker/data.build.wanda.yaml
 ci/docker/datan.build.wanda.yaml
 ci/docker/data9.build.wanda.yaml
 ci/docker/datal.build.wanda.yaml
-@ data ml train linux_wheels
+@ data ml train
 ;
 
 python/ray/workflow/

ci/ray_ci/BUILD.bazel

@@ -9,15 +9,19 @@ py_library(
             "test_*.py",
             "test_in_docker.py",
             "build_in_docker.py",
+            "build_in_docker_windows.py",
         ],
     ),
     data = glob(["*.yaml"]),
     visibility = ["//ci/ray_ci:__subpackages__"],
     deps = [
+        "//release:bazel",
+        "//release:global_config",
+        "//release:test",
+        "//release:test_automation",
         ci_require("boto3"),
         ci_require("pyyaml"),
         ci_require("click"),
-        "//release:ray_release",
     ],
 )
 

ci/raydepsets/BUILD.bazel

@@ -43,6 +43,7 @@ py_test(
         "tests/test_data/requirements_compiled_test_update.txt",
         "tests/test_data/requirements_test.txt",
         "tests/test_data/test.depsets.yaml",
+        "tests/test_data/test2.depsets.yaml",
     ],
     exec_compatible_with = ["//:hermetic_python"],
     tags = [

ci/raydepsets/cli.py

@@ -1,4 +1,5 @@
 import difflib
+import os
 import platform
 import shlex
 import shutil
@@ -90,20 +91,22 @@ def __init__(
         check: Optional[bool] = False,
     ):
         self.workspace = Workspace(workspace_dir)
-        self.config = self.workspace.load_config(config_path)
-        if check:
-            self.temp_dir = tempfile.mkdtemp()
-            self.output_paths = self.get_output_paths()
-            self.copy_to_temp_dir()
+        self.config = self.workspace.load_configs(config_path)
+        self.config_name = os.path.basename(config_path)
         self.build_graph = DiGraph()
         self._build()
         self._uv_binary = _uv_binary()
         self._uv_cache_dir = uv_cache_dir
+        if check:
+            self.temp_dir = tempfile.mkdtemp()
+            self.output_paths = self.get_output_paths()
+            self.copy_to_temp_dir()
 
     def get_output_paths(self) -> List[Path]:
         output_paths = []
-        for depset in self.config.depsets:
-            output_paths.append(Path(depset.output))
+        for node in topological_sort(self.build_graph):
+            if self.build_graph.nodes[node]["node_type"] == "depset":
+                output_paths.append(Path(self.build_graph.nodes[node]["depset"].output))
         return output_paths
 
     def copy_to_temp_dir(self):
@@ -148,16 +151,28 @@ def _build(self):
         for depset in self.config.depsets:
             if depset.operation == "compile":
                 self.build_graph.add_node(
-                    depset.name, operation="compile", depset=depset, node_type="depset"
+                    depset.name,
+                    operation="compile",
+                    depset=depset,
+                    node_type="depset",
+                    config_name=depset.config_name,
                 )
             elif depset.operation == "subset":
                 self.build_graph.add_node(
-                    depset.name, operation="subset", depset=depset, node_type="depset"
+                    depset.name,
+                    operation="subset",
+                    depset=depset,
+                    node_type="depset",
+                    config_name=depset.config_name,
                 )
                 self.build_graph.add_edge(depset.source_depset, depset.name)
             elif depset.operation == "expand":
                 self.build_graph.add_node(
-                    depset.name, operation="expand", depset=depset, node_type="depset"
+                    depset.name,
+                    operation="expand",
+                    depset=depset,
+                    node_type="depset",
+                    config_name=depset.config_name,
                 )
                 for depset_name in depset.depsets:
                     self.build_graph.add_edge(depset_name, depset.name)
@@ -171,20 +186,39 @@ def _build(self):
                         operation="pre_hook",
                         pre_hook=hook,
                         node_type="pre_hook",
+                        config_name=depset.config_name,
                     )
                     self.build_graph.add_edge(hook_name, depset.name)
+        self.subgraph_config_nodes()
 
     def subgraph_dependency_nodes(self, depset_name: str):
         dependency_nodes = networkx_ancestors(self.build_graph, depset_name)
         nodes = dependency_nodes | {depset_name}
         self.build_graph = self.build_graph.subgraph(nodes).copy()
 
+    def subgraph_config_nodes(self):
+        # Get all nodes that have the target config name
+        config_nodes = [
+            node
+            for node in self.build_graph.nodes
+            if self.build_graph.nodes[node]["config_name"] == self.config_name
+        ]
+        # Get all ancestors of the target config nodes
+        ancestors_by_confg_node = {
+            n: networkx_ancestors(self.build_graph, n) for n in config_nodes
+        }
+        # Union all the ancestors of the target config nodes
+        config_nodes_ancestors = set().union(
+            *(ancestors_by_confg_node[n] for n in config_nodes)
+        )
+        nodes = set(config_nodes) | config_nodes_ancestors
+        self.build_graph = self.build_graph.subgraph(nodes).copy()
+
     def execute(self, single_depset_name: Optional[str] = None):
         if single_depset_name:
             # check if the depset exists
             _get_depset(self.config.depsets, single_depset_name)
             self.subgraph_dependency_nodes(single_depset_name)
-
         for node in topological_sort(self.build_graph):
             node_type = self.build_graph.nodes[node]["node_type"]
             if node_type == "pre_hook":