[core] Fix flaky recursive cancel test (#58923)

This test was flaky before, making it more deterministic now by
cancelling only once the outer and inner task are both running and
making sure they're running before launching the many_resources task.

The flaky failure was the ray.get on many_resources returning before the
cancel.

https://buildkite.com/ray-project/postmerge/builds/14463#019a99a6-acf2-4e28-89d3-2abc99eb93ac/609-913

---------

Signed-off-by: dayshah <dhyey2019@gmail.com>

Author: dayshah

python/ray/tests/test_cancel.py

@@ -458,29 +458,32 @@ def remote_wait(sg):
 
 @pytest.mark.parametrize("use_force", [True, False])
 def test_recursive_cancel(shutdown_only, use_force):
-    ray.init(num_cpus=4)
+    ray.init(num_cpus=2)
 
     @ray.remote(num_cpus=1)
-    def inner():
+    def inner(signal_actor):
+        signal_actor.send.remote()
         while True:
             time.sleep(0.1)
 
     @ray.remote(num_cpus=1)
-    def outer():
-        x = [inner.remote()]
-        print(x)
+    def outer(signal_actor):
+        _ = inner.remote(signal_actor)
         while True:
             time.sleep(0.1)
 
-    @ray.remote(num_cpus=4)
+    @ray.remote(num_cpus=2)
     def many_resources():
-        return 300
+        return True
 
-    outer_fut = outer.remote()
+    signal_actor = SignalActor.remote()
+    outer_fut = outer.remote(signal_actor)
+    # Wait until both inner and outer are running
+    ray.get(signal_actor.wait.remote())
     many_fut = many_resources.remote()
     with pytest.raises(GetTimeoutError):
         ray.get(many_fut, timeout=1)
-    ray.cancel(outer_fut)
+    ray.cancel(outer_fut, force=use_force)
     with pytest.raises(valid_exceptions(use_force)):
         ray.get(outer_fut, timeout=10)