[Serve] Fix serve deploy not getting current directory added to sys.path (#43899)

Re: #43214 There is a core change that stopped adding script directory and current directory for different conditions. This accidentally stopped adding current directory when it's coming from the dashboard where in that case we only want to stop adding the script directory. This PR fixes the issue and allow serve deploy to continue working with the current directory.

---------

Signed-off-by: Gene Su <e870252314@gmail.com>

Author: GeneDer

python/ray/_private/worker.py

@@ -2348,7 +2348,8 @@ def connect(
         # Formally, the local path(s) should be added when all of these are true:
         # (1) there's no working_dir (or code search path should be in working_dir),
         # (2) it's not interactive mode, (there's no script file in interactive mode),
-        # (3) it's not in dashboard,
+        # (3) it's not in dashboard (should only skip script location but still append
+        #   current directory),
         # (4) it's not client mode, (handled by client code)
         # (5) the driver is at the same node (machine) as the worker.
         #
@@ -2358,8 +2359,6 @@ def connect(
             [
                 job_config._runtime_env_has_working_dir(),
                 interactive_mode,
-                namespace
-                and namespace == ray_constants.RAY_INTERNAL_DASHBOARD_NAMESPACE,
                 job_config._client_job,
             ]
         ):
@@ -2370,7 +2369,13 @@ def connect(
             # see https://peps.python.org/pep-0338/),
             # then we shouldn't add it to the workers.
             if script_directory in sys.path:
-                code_paths.add(script_directory)
+                # If user code contains names like `utils`, it will be conflicting with
+                # the `utils` module in the dashboard module. We should skip adding
+                # script directory for dashboard clients and should have the
+                # current directory available for loading user code.
+                # See: https://github.com/anyscale/product/issues/20746
+                if namespace != ray_constants.RAY_INTERNAL_DASHBOARD_NAMESPACE:
+                    code_paths.add(script_directory)
             current_directory = os.path.abspath(os.path.curdir)
             code_paths.add(current_directory)
             if len(code_paths) != 0:

python/ray/serve/tests/conftest.py

@@ -151,6 +151,35 @@ def ray_start_stop():
     )
 
 
+@pytest.fixture(scope="function")
+def ray_start_stop_in_specific_directory(request):
+    original_working_dir = os.getcwd()
+
+    # Change working directory so Ray will start in the requested directory.
+    new_working_dir = request.param
+    os.chdir(new_working_dir)
+    print(f"\nChanged working directory to {new_working_dir}\n")
+
+    subprocess.check_output(["ray", "start", "--head"])
+    wait_for_condition(
+        lambda: requests.get("http://localhost:52365/api/ray/version").status_code
+        == 200,
+        timeout=15,
+    )
+    try:
+        yield
+    finally:
+        # Change the directory back to the original one.
+        os.chdir(original_working_dir)
+        print(f"\nChanged working directory back to {original_working_dir}\n")
+
+        subprocess.check_output(["ray", "stop", "--force"])
+        wait_for_condition(
+            check_ray_stop,
+            timeout=15,
+        )
+
+
 @pytest.fixture
 def ray_instance(request):
     """Starts and stops a Ray instance for this test.

python/ray/serve/tests/test_cli.py

@@ -700,5 +700,34 @@ def test_deploy_from_import_path(ray_start_stop):
     )
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
+@pytest.mark.parametrize(
+    "ray_start_stop_in_specific_directory",
+    [
+        os.path.join(os.path.dirname(__file__), "test_config_files"),
+    ],
+    indirect=True,
+)
+def test_deploy_with_access_to_current_directory(ray_start_stop_in_specific_directory):
+    """Test serve deploy using modules in the current directory succeeds.
+
+    There was an issue where dashboard client doesn't add the current directory to
+    the sys.path and failed to deploy a Serve app defined in the directory. This
+    test ensures that files in the current directory can be accessed and deployed.
+
+    See: https://github.com/ray-project/ray/issues/43889
+    """
+    # Deploy Serve application with a config in the current directory.
+    subprocess.check_output(["serve", "deploy", "use_current_working_directory.yaml"])
+
+    # Ensure serve deploy eventually succeeds.
+    def check_deploy_successfully():
+        status_response = subprocess.check_output(["serve", "status"])
+        assert b"RUNNING" in status_response
+        return True
+
+    wait_for_condition(check_deploy_successfully, timeout=5)
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main(["-v", "-s", __file__]))

python/ray/serve/tests/test_config_files/use_current_working_directory.py

@@ -0,0 +1,9 @@
+from ray import serve
+
+
+@serve.deployment
+def f():
+    return "hi"
+
+
+app = f.bind()

python/ray/serve/tests/test_config_files/use_current_working_directory.yaml

@@ -0,0 +1,6 @@
+applications:
+  - name: app1
+    route_prefix: /
+    import_path: use_current_working_directory:app
+    deployments:
+      - name: f