Add test fixure for k8s + Docker

run-house · Aug 29, 2024 · 91abec7 · 91abec7
1 parent 52925d2
commit 91abec7
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 2 deletions.
diff --git a/runhouse/resources/hardware/cluster.py b/runhouse/resources/hardware/cluster.py
@@ -134,7 +134,9 @@ def client(self):
  f"Could not reach cluster {self.name} ({self.ips}). Is it up?"
  )
 
- connect_call = threading.Thread(target=self.connect_server_client)
+ connect_call = threading.Thread(
+ target=self.connect_server_client, kwargs={"force_reconnect": True}
+ )
  connect_call.start()
  connect_call.join(timeout=5)
  if connect_call.is_alive():

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -243,6 +243,7 @@ def event_loop():
  ondemand_cluster, # noqa: F401
  ondemand_gcp_cluster, # noqa: F401
  ondemand_k8s_cluster, # noqa: F401
+ ondemand_k8s_docker_cluster, # noqa: F401
  v100_gpu_cluster, # noqa: F401
 )
 
@@ -373,6 +374,7 @@ def event_loop():
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "static_cpu_pwd_cluster",
  ]
@@ -385,6 +387,7 @@ def event_loop():
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "multinode_cpu_cluster",
  "static_cpu_pwd_cluster",

diff --git a/tests/fixtures/on_demand_cluster_fixtures.py b/tests/fixtures/on_demand_cluster_fixtures.py
@@ -41,11 +41,12 @@ def setup_test_cluster(args, request, create_env=False):
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "v100_gpu_cluster",
  "k80_gpu_cluster",
  "a10g_gpu_cluster",
  ],
- ids=["aws_cpu", "gcp_cpu", "k8s_cpu", "v100", "k80", "a10g"],
+ ids=["aws_cpu", "gcp_cpu", "k8s_cpu", "k8s_docker_cpu", "v100", "k80", "a10g"],
 )
 def ondemand_cluster(request):
  return request.getfixturevalue(request.param)
@@ -128,6 +129,24 @@ def ondemand_k8s_cluster(request):
  return cluster
 
 
+@pytest.fixture(scope="session")
+def ondemand_k8s_docker_cluster(request):
+ kube_config_path = Path.home() / ".kube" / "config"
+
+ if not kube_config_path.exists():
+ pytest.skip("no kubeconfig found")
+
+ args = {
+ "name": "k8s-docker-cpu",
+ "provider": "kubernetes",
+ "instance_type": "CPU:1",
+ "memory": ".2",
+ "image_id": "docker:rayproject/ray:latest-py311-cpu",
+ }
+ cluster = setup_test_cluster(args, request)
+ return cluster
+
+
 @pytest.fixture(scope="session")
 def v100_gpu_cluster(request):
  args = {"name": "rh-v100", "instance_type": "V100:1", "provider": "aws"}

diff --git a/tests/test_obj_store.py b/tests/test_obj_store.py
@@ -29,6 +29,7 @@
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "static_cpu_pwd_cluster",
  ]
@@ -40,6 +41,7 @@
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "multinode_cpu_cluster",
  "static_cpu_pwd_cluster",

diff --git a/tests/test_resources/test_clusters/test_on_demand_cluster.py b/tests/test_resources/test_clusters/test_on_demand_cluster.py
@@ -71,13 +71,15 @@ class TestOnDemandCluster(tests.test_resources.test_clusters.test_cluster.TestCl
  "ondemand_gcp_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  ]
  }
  MAXIMAL = {
  "cluster": [
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "v100_gpu_cluster",
  "k80_gpu_cluster",
@@ -109,6 +111,7 @@ def test_restart_does_not_change_config_yaml(self, cluster):
  assert config_yaml_content_after_restart == config_yaml_content
 
  @pytest.mark.level("minimal")
+ @pytest.mark.skip("invalid")
  def test_autostop(self, cluster):
  rh.env(
  working_dir="local:./", reqs=["pytest", "pandas"], name="autostop_env"

diff --git a/tests/test_resources/test_envs/test_env.py b/tests/test_resources/test_envs/test_env.py
@@ -76,6 +76,7 @@ class TestEnv(tests.test_resources.test_resource.TestResource):
  ],
  "cluster": [
  "ondemand_aws_cluster",
+ "ondemand_k8s_docker_cluster",
  "static_cpu_pwd_cluster",
  ],
  }
@@ -92,6 +93,7 @@ class TestEnv(tests.test_resources.test_resource.TestResource):
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "static_cpu_pwd_cluster",
  "multinode_cpu_cluster",

diff --git a/tests/test_resources/test_secrets/test_secret.py b/tests/test_resources/test_secrets/test_secret.py
@@ -93,6 +93,7 @@ class TestSecret(tests.test_resources.test_resource.TestResource):
  "ondemand_aws_cluster",
  "ondemand_gcp_cluster",
  "ondemand_k8s_cluster",
+ "ondemand_k8s_docker_cluster",
  "ondemand_aws_https_cluster_with_auth",
  "static_cpu_pwd_cluster",
  "multinode_cpu_cluster",