[ci][core] Mark python tests flaky on CI

python/ray/_private/test_utils.py

@@ -1979,3 +1979,17 @@ def reset_autoscaler_v2_enabled_cache():
     import ray.autoscaler.v2.utils as u
 
     u.cached_is_autoscaler_v2 = None
+
+
+def skip_flaky_test() -> bool:
+    """
+    Skip a test if it is flaky (e.g. in premerge)
+
+    Default we will skip the flaky test if not specified otherwise in
+    CI with CI_SKIP_FLAKY_TEST="0"
+
+
+    Returns:
+        bool: True if the test should be skipped
+    """
+    return os.environ.get("CI_SKIP_FLAKY_TEST", "1") == "1"

python/ray/tests/test_client_builder.py

@@ -13,6 +13,7 @@
     run_string_as_driver,
     run_string_as_driver_nonblocking,
     wait_for_condition,
+    skip_flaky_test,
 )
 from ray.util.state import list_workers
 
@@ -327,6 +328,9 @@ def has_client_deprecation_warn(warning: Warning, expected_replacement: str) ->
 @pytest.mark.filterwarnings(
     "default:Starting a connection through `ray.client` will be deprecated"
 )
+@pytest.mark.skipif(
+    skip_flaky_test(), reason="https://github.com/ray-project/ray/issues/38224"
+)
 def test_client_deprecation_warn():
     """
     Tests that calling ray.client directly raises a deprecation warning with

python/ray/tests/test_failure_3.py

@@ -16,6 +16,7 @@
     wait_for_pid_to_exit,
     wait_for_condition,
     run_string_as_driver_nonblocking,
+    skip_flaky_test,
 )
 
 SIGKILL = signal.SIGKILL if sys.platform != "win32" else signal.SIGTERM
@@ -240,6 +241,10 @@ def new_task(_):
     [{"_system_config": {"timeout_ms_task_wait_for_death_info": 100000000}}],
     indirect=True,
 )
+@pytest.mark.skipif(
+    skip_flaky_test(),
+    reason="https://github.com/ray-project/ray/issues/41188",
+)
 def test_actor_failure_async_3(ray_start_regular):
     @ray.remote(max_restarts=1)
     class A:

python/ray/tests/test_object_assign_owner.py

@@ -3,6 +3,7 @@
 import time
 import numpy as np
 import os
+from ray._private.test_utils import skip_flaky_test
 
 
 # https://github.com/ray-project/ray/issues/19659
@@ -155,6 +156,9 @@ def get_objects(self, refs):
 
 
 # https://github.com/ray-project/ray/issues/30341
+@pytest.mark.skipif(
+    skip_flaky_test(), reason="https://github.com/ray-project/ray/issues/41175"
+)
 def test_owner_assign_inner_object(shutdown_only):
 
     ray.init()

python/ray/tests/test_placement_group.py

@@ -1,12 +1,14 @@
 import sys
 import warnings
+import os
 
 import pytest
 
 import ray
 from ray._private.utils import get_ray_doc_version
 import ray.cluster_utils
 from ray._private.test_utils import placement_group_assert_no_leak
+from ray._private.test_utils import skip_flaky_test
 from ray.util.client.ray_client_helpers import connect_to_client_or_not
 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
 
@@ -338,6 +340,10 @@ def value(self):
 
 @pytest.mark.parametrize("connect_to_client", [False, True])
 @pytest.mark.parametrize("gcs_actor_scheduling_enabled", [False, True])
+@pytest.mark.skipif(
+    skip_flaky_test(),
+    reason="https://github.com/ray-project/ray/issues/38726",
+)
 def test_placement_group_strict_spread(
     ray_start_cluster, connect_to_client, gcs_actor_scheduling_enabled
 ):
@@ -646,8 +652,6 @@ def test_omp_num_threads_in_pg(ray_start_cluster):
 
     @ray.remote(num_cpus=3)
     def test_omp_num_threads():
-        import os
-
         omp_threads = os.environ["OMP_NUM_THREADS"]
         return int(omp_threads)
 
@@ -670,8 +674,6 @@ def test_omp_num_threads():
 
 
 if __name__ == "__main__":
-    import os
-
     if os.environ.get("PARALLEL_CI"):
         sys.exit(pytest.main(["-n", "auto", "--boxed", "-vs", __file__]))
     else:

python/ray/tests/test_runtime_env_working_dir_3.py

@@ -12,6 +12,7 @@
 import ray.experimental.internal_kv as kv
 from ray._private.ray_constants import RAY_RUNTIME_ENV_URI_PIN_EXPIRATION_S_ENV_VAR
 from ray._private.test_utils import chdir, check_local_files_gced, wait_for_condition
+from ray._private.test_utils import skip_flaky_test
 from ray._private.utils import get_directory_size_bytes
 
 # This test requires you have AWS credentials set up (any AWS credentials will
@@ -249,6 +250,10 @@ def check(self):
     @pytest.mark.parametrize(
         "source", [S3_PACKAGE_URI, lazy_fixture("tmp_working_dir")]
     )
+    @pytest.mark.skipif(
+        skip_flaky_test(),
+        reason="https://github.com/ray-project/ray/issues/40781",
+    )
     def test_detached_actor_gc(
         self,
         start_cluster,

python/ray/tests/test_state_api_log.py

@@ -20,6 +20,7 @@
     format_web_url,
     wait_for_condition,
     wait_until_server_available,
+    skip_flaky_test,
 )
 
 from ray._private.ray_constants import (
@@ -1272,6 +1273,10 @@ def verify():
 @pytest.mark.skipif(
     sys.platform == "win32", reason="Windows has logging race from tasks."
 )
+@pytest.mark.skipif(
+    skip_flaky_test(),
+    reason="https://github.com/ray-project/ray/issues/40959",
+)
 def test_log_task(shutdown_only):
     from ray.runtime_env import RuntimeEnv