[CI] Repeat start_instance (#361)

Recently frequently occurs a fail of regression due to fail
start_instance due to "Insufficient capacity".

Repeat attempts to start instances 300 times with 60 seconds sleep between repeats.

Tested here
https://github.com/CentML/hidet/actions/runs/10000711025/job/27664169588

.github/scripts/start_instances.py

@@ -97,14 +97,19 @@ def run_command(cmd):
 
     # Start all instances
     for instance in instances:
-        cloud_provider_id, instance_id, _ = instance
-        if cloud_provider_id == 1: # AWS
-            cmd = ['aws', 'ec2', 'start-instances', '--instance-ids', instance_id]
-        elif cloud_provider_id == 2: # Always on, no need to launch. Do Nothing.
-            cmd = ['true']
-        else:
-            raise ValueError(f'Unknown cloud provider id: {cloud_provider_id}')
-        output = run_command(cmd)
+        for i in range(300):
+            cloud_provider_id, instance_id, _ = instance
+            if cloud_provider_id == 1: # AWS
+                cmd = ['aws', 'ec2', 'start-instances', '--instance-ids', instance_id]
+            elif cloud_provider_id == 2: # Always on, no need to launch. Do Nothing.
+                cmd = ['true']
+            else:
+                raise ValueError(f'Unknown cloud provider id: {cloud_provider_id}')
+            output = run_command(cmd)
+            if output.returncode == 0:
+                break
+            time.sleep(60)
+
         if output.returncode != 0:
             raise RuntimeError(f'Failed to start instance {instance_id} on cloud provider {cloud_provider_id}.')
 

.github/workflows/regression.yaml

@@ -34,7 +34,7 @@ jobs:
 
       - name: Run main Python script
         id: run_py_script
-        run: timeout 900 python ./.github/scripts/start_instances.py
+        run: timeout 36000 python ./.github/scripts/start_instances.py
         env:
           # TODO: Allow launching only specified GPU instances
           HW_CONFIG: all