[AIR] Bump start timeout for Horovod Release Tests (#33083)

Horovod on Ray release tests have been timing out during the initial rendezvous. This PR bumps the timeout configuration from 30 seconds to 120 seconds. --------- Signed-off-by: amogkam <amogkamsetty@yahoo.com>
ray-project · Mar 7, 2023 · e0fda4a · e0fda4a
1 parent c0f6068
commit e0fda4a
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/release/ml_user_tests/horovod/horovod_user_test.py b/release/ml_user_tests/horovod/horovod_user_test.py
@@ -22,6 +22,7 @@
         num_workers=6,
         use_gpu=True,
         placement_group_timeout_s=2000,
+        timeout_s=120,
         kwargs={"num_epochs": 20},
     )
 

diff --git a/release/nightly_tests/dataset/pipelined_training.py b/release/nightly_tests/dataset/pipelined_training.py
@@ -327,7 +327,7 @@ def consume(split, rank=None, batch_size=None):
         ray.get(tasks)
     else:
         print("Create Ray executor")
-        settings = RayExecutor.create_settings(timeout_s=30)
+        settings = RayExecutor.create_settings(timeout_s=120)
         executor = RayExecutor(settings, num_workers=args.num_workers, use_gpu=True)
         executor.start()
         executor.run(train_main, args=[args, splits])