add test fixutre for k8s and docker

run-house · Aug 29, 2024 · a99e2c7 · a99e2c7
1 parent 017c197
commit a99e2c7
Show file tree

Hide file tree

Showing 7 changed files with 33 additions and 2 deletions.
diff --git a/runhouse/resources/hardware/cluster.py b/runhouse/resources/hardware/cluster.py
@@ -134,7 +134,9 @@ def client(self):
                     f"Could not reach cluster {self.name} ({self.ips}). Is it up?"
                 )
 
-            connect_call = threading.Thread(target=self.connect_server_client)
+            connect_call = threading.Thread(
+                target=self.connect_server_client, kwargs={"force_reconnect": True}
+            )
             connect_call.start()
             connect_call.join(timeout=5)
             if connect_call.is_alive():

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -243,6 +243,7 @@ def event_loop():
     ondemand_cluster,  # noqa: F401
     ondemand_gcp_cluster,  # noqa: F401
     ondemand_k8s_cluster,  # noqa: F401
+    ondemand_k8s_docker_cluster,  # noqa: F401
     v100_gpu_cluster,  # noqa: F401
 )
 
@@ -373,6 +374,7 @@ def event_loop():
         "ondemand_aws_cluster",
         "ondemand_gcp_cluster",
         "ondemand_k8s_cluster",
+        "ondemand_k8s_docker_cluster",
         "ondemand_aws_https_cluster_with_auth",
         "static_cpu_pwd_cluster",
     ]
@@ -385,6 +387,7 @@ def event_loop():
         "ondemand_aws_cluster",
         "ondemand_gcp_cluster",
         "ondemand_k8s_cluster",
+        "ondemand_k8s_docker_cluster",
         "ondemand_aws_https_cluster_with_auth",
         "multinode_cpu_cluster",
         "static_cpu_pwd_cluster",

diff --git a/tests/fixtures/on_demand_cluster_fixtures.py b/tests/fixtures/on_demand_cluster_fixtures.py
@@ -41,11 +41,12 @@ def setup_test_cluster(args, request, create_env=False):
         "ondemand_aws_cluster",
         "ondemand_gcp_cluster",
         "ondemand_k8s_cluster",
+        "ondemand_k8s_docker_cluster",
         "v100_gpu_cluster",
         "k80_gpu_cluster",
         "a10g_gpu_cluster",
     ],
-    ids=["aws_cpu", "gcp_cpu", "k8s_cpu", "v100", "k80", "a10g"],
+    ids=["aws_cpu", "gcp_cpu", "k8s_cpu", "k8s_docker_cpu", "v100", "k80", "a10g"],
 )
 def ondemand_cluster(request):
     return request.getfixturevalue(request.param)
@@ -128,6 +129,24 @@ def ondemand_k8s_cluster(request):
     return cluster
 
 
+@pytest.fixture(scope="session")
+def ondemand_k8s_docker_cluster(request):
+    kube_config_path = Path.home() / ".kube" / "config"
+
+    if not kube_config_path.exists():
+        pytest.skip("no kubeconfig found")
+
+    args = {
+        "name": "k8s-docker-cpu",
+        "provider": "kubernetes",
+        "instance_type": "CPU:1",
+        "memory": ".2",
+        "image_id": "docker:rayproject/ray:latest-py311-cpu",
+    }
+    cluster = setup_test_cluster(args, request)
+    return cluster
+
+
 @pytest.fixture(scope="session")
 def v100_gpu_cluster(request):
     args = {"name": "rh-v100", "instance_type": "V100:1", "provider": "aws"}

diff --git a/tests/test_obj_store.py b/tests/test_obj_store.py
@@ -29,6 +29,7 @@
         "ondemand_aws_cluster",
         "ondemand_gcp_cluster",
         "ondemand_k8s_cluster",
+        "ondemand_k8s_docker_cluster",
         "ondemand_aws_https_cluster_with_auth",
         "static_cpu_pwd_cluster",
     ]
@@ -40,6 +41,7 @@
         "ondemand_aws_cluster",
         "ondemand_gcp_cluster",
         "ondemand_k8s_cluster",
+        "ondemand_k8s_docker_cluster",
         "ondemand_aws_https_cluster_with_auth",
         "multinode_cpu_cluster",
         "static_cpu_pwd_cluster",

diff --git a/tests/test_resources/test_clusters/test_on_demand_cluster.py b/tests/test_resources/test_clusters/test_on_demand_cluster.py
@@ -71,13 +71,15 @@ class TestOnDemandCluster(tests.test_resources.test_clusters.test_cluster.TestCl
             "ondemand_gcp_cluster",
             "ondemand_aws_https_cluster_with_auth",
             "ondemand_k8s_cluster",
+            "ondemand_k8s_docker_cluster",
         ]
     }
     MAXIMAL = {
         "cluster": [
             "ondemand_aws_cluster",
             "ondemand_gcp_cluster",
             "ondemand_k8s_cluster",
+            "ondemand_k8s_docker_cluster",
             "ondemand_aws_https_cluster_with_auth",
             "v100_gpu_cluster",
             "k80_gpu_cluster",

diff --git a/tests/test_resources/test_envs/test_env.py b/tests/test_resources/test_envs/test_env.py
@@ -76,6 +76,7 @@ class TestEnv(tests.test_resources.test_resource.TestResource):
         ],
         "cluster": [
             "ondemand_aws_cluster",
+            "ondemand_k8s_docker_cluster",
             "static_cpu_pwd_cluster",
         ],
     }
@@ -92,6 +93,7 @@ class TestEnv(tests.test_resources.test_resource.TestResource):
             "ondemand_aws_cluster",
             "ondemand_gcp_cluster",
             "ondemand_k8s_cluster",
+            "ondemand_k8s_docker_cluster",
             "ondemand_aws_https_cluster_with_auth",
             "static_cpu_pwd_cluster",
             "multinode_cpu_cluster",

diff --git a/tests/test_resources/test_secrets/test_secret.py b/tests/test_resources/test_secrets/test_secret.py
@@ -93,6 +93,7 @@ class TestSecret(tests.test_resources.test_resource.TestResource):
             "ondemand_aws_cluster",
             "ondemand_gcp_cluster",
             "ondemand_k8s_cluster",
+            "ondemand_k8s_docker_cluster",
             "ondemand_aws_https_cluster_with_auth",
             "static_cpu_pwd_cluster",
             "multinode_cpu_cluster",