From 37e1b3d59be2bc6cf978c60ff128d5764a9057c5 Mon Sep 17 00:00:00 2001 From: dongreenberg Date: Mon, 17 Jun 2024 14:26:57 -0400 Subject: [PATCH] Consolidate static cluster fixtures into one --- tests/conftest.py | 9 +-- tests/fixtures/static_cluster_fixtures.py | 65 +++---------------- tests/test_obj_store.py | 2 - .../test_clusters/test_cluster.py | 2 - .../test_clusters/test_on_demand_cluster.py | 1 - tests/test_resources/test_envs/test_env.py | 3 +- .../test_secrets/test_secret.py | 3 +- 7 files changed, 12 insertions(+), 73 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 80538b8f0..8161d0a18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,7 +43,6 @@ class TestCluster(tests.test_resources.test_resource.TestResource): "cluster": [ "ondemand_aws_cluster", "static_cpu_cluster", - "password_cluster", ] } MAXIMAL = { @@ -53,7 +52,6 @@ class TestCluster(tests.test_resources.test_resource.TestResource): "docker_cluster_pwd_ssh_no_auth", "ondemand_aws_cluster", "static_cpu_cluster", - "password_cluster", "multinode_cpu_cluster" ] } @@ -260,10 +258,7 @@ def event_loop(): unnamed_resource, # noqa: F401 ) -from tests.fixtures.static_cluster_fixtures import ( - password_cluster, # noqa: F401 - static_cpu_cluster, # noqa: F401 -) +from tests.fixtures.static_cluster_fixtures import static_cpu_cluster # noqa: F401 from tests.test_resources.test_clusters.test_sagemaker_cluster.conftest import ( other_sm_cluster, # noqa: F401 @@ -390,7 +385,6 @@ def event_loop(): "ondemand_gcp_cluster", "ondemand_k8s_cluster", "ondemand_aws_https_cluster_with_auth", - "password_cluster", "static_cpu_cluster", ] } @@ -403,7 +397,6 @@ def event_loop(): "ondemand_gcp_cluster", "ondemand_k8s_cluster", "ondemand_aws_https_cluster_with_auth", - "password_cluster", "multinode_cpu_cluster", "static_cpu_cluster", "multinode_gpu_cluster", # for testing cluster status on multinode gpu. diff --git a/tests/fixtures/static_cluster_fixtures.py b/tests/fixtures/static_cluster_fixtures.py index b55c153b9..6be9466c0 100644 --- a/tests/fixtures/static_cluster_fixtures.py +++ b/tests/fixtures/static_cluster_fixtures.py @@ -8,60 +8,7 @@ @pytest.fixture(scope="session") def static_cpu_cluster(): - # TODO: Spin up a new basic m5.xlarge EC2 instance - # import boto3 - - # ec2 = boto3.resource("ec2") - # instances = ec2.create_instances( - # ImageId="ami-0a313d6098716f372", - # InstanceType="m5.xlarge", - # MinCount=1, - # MaxCount=1, - # KeyName="sky-key", - # TagSpecifications=[ - # { - # "ResourceType": "instance", - # "Tags": [ - # {"Key": "Name", "Value": "rh-cpu"}, - # ], - # }, - # ], - # ) - # instance = instances[0] - # instance.wait_until_running() - # instance.load() - - # ip = instance.public_ip_address - - c = ( - rh.ondemand_cluster( - instance_type="m5.xlarge", - provider="aws", - region="us-east-1", - # image_id="ami-0a313d6098716f372", # Upgraded to python 3.11.4 which is not compatible with ray 2.4.0 - name="test-byo-cluster", - ) - .up_if_not() - .save() - ) - - args = dict( - name="different-cluster", - host=c.address, - ssh_creds={"ssh_user": "ubuntu", "ssh_private_key": "~/.ssh/sky-key"}, - ) - c = rh.cluster(**args).save() - c.restart_server(resync_rh=True) # needed to override the cluster's config file - init_args[id(c)] = args - - test_env().to(c) - - return c - - -@pytest.fixture(scope="session") -def password_cluster(): - sky_cluster = rh.cluster("temp-rh-password", instance_type="CPU:4").save() + sky_cluster = rh.cluster("aws-cpu-password", instance_type="CPU:4").save() if not sky_cluster.is_up(): sky_cluster.up() @@ -83,8 +30,14 @@ def password_cluster(): sky_cluster.run(["rm -rf runhouse/"]) # instantiate byo cluster with password - ssh_creds = {"ssh_user": "ubuntu", "password": "cluster-pass"} - args = dict(name="rh-password", host=[sky_cluster.address], ssh_creds=ssh_creds) + ssh_creds = { + "ssh_user": "ubuntu", + "ssh_private_key": "~/.ssh/sky-key", + "password": "cluster-pass", + } + args = dict( + name="static-cpu-password", host=[sky_cluster.address], ssh_creds=ssh_creds + ) c = rh.cluster(**args).save() c.restart_server(resync_rh=True) init_args[id(c)] = args diff --git a/tests/test_obj_store.py b/tests/test_obj_store.py index c553bed69..51fe360ea 100644 --- a/tests/test_obj_store.py +++ b/tests/test_obj_store.py @@ -30,7 +30,6 @@ "ondemand_gcp_cluster", "ondemand_k8s_cluster", "ondemand_aws_https_cluster_with_auth", - "password_cluster", "static_cpu_cluster", ] } @@ -42,7 +41,6 @@ "ondemand_gcp_cluster", "ondemand_k8s_cluster", "ondemand_aws_https_cluster_with_auth", - "password_cluster", "multinode_cpu_cluster", "static_cpu_cluster", ] diff --git a/tests/test_resources/test_clusters/test_cluster.py b/tests/test_resources/test_clusters/test_cluster.py index b4b8102c1..c99bbbaef 100644 --- a/tests/test_resources/test_clusters/test_cluster.py +++ b/tests/test_resources/test_clusters/test_cluster.py @@ -108,7 +108,6 @@ class TestCluster(tests.test_resources.test_resource.TestResource): RELEASE = { "cluster": [ "static_cpu_cluster", - "password_cluster", ] } MAXIMAL = { @@ -117,7 +116,6 @@ class TestCluster(tests.test_resources.test_resource.TestResource): "docker_cluster_pk_ssh_den_auth", "docker_cluster_pwd_ssh_no_auth", "static_cpu_cluster", - "password_cluster", "multinode_cpu_cluster", ] } diff --git a/tests/test_resources/test_clusters/test_on_demand_cluster.py b/tests/test_resources/test_clusters/test_on_demand_cluster.py index 92a08ff9f..f11a44f35 100644 --- a/tests/test_resources/test_clusters/test_on_demand_cluster.py +++ b/tests/test_resources/test_clusters/test_on_demand_cluster.py @@ -73,7 +73,6 @@ class TestOnDemandCluster(tests.test_resources.test_clusters.test_cluster.TestCl "k80_gpu_cluster", "a10g_gpu_cluster", "static_cpu_cluster", - "password_cluster", "multinode_cpu_cluster", "multinode_gpu_cluster", ] diff --git a/tests/test_resources/test_envs/test_env.py b/tests/test_resources/test_envs/test_env.py index 53346e564..2036c4d7f 100644 --- a/tests/test_resources/test_envs/test_env.py +++ b/tests/test_resources/test_envs/test_env.py @@ -70,7 +70,7 @@ class TestEnv(tests.test_resources.test_resource.TestResource): ], "cluster": [ "ondemand_aws_cluster", - "password_cluster", + "static_cpu_cluster", ], } MAXIMAL = { @@ -88,7 +88,6 @@ class TestEnv(tests.test_resources.test_resource.TestResource): "ondemand_k8s_cluster", "ondemand_aws_https_cluster_with_auth", "static_cpu_cluster", - "password_cluster", "multinode_cpu_cluster", "docker_cluster_pk_ssh_no_auth", "docker_cluster_pwd_ssh_no_auth", diff --git a/tests/test_resources/test_secrets/test_secret.py b/tests/test_resources/test_secrets/test_secret.py index c1cfad2e7..d98a7dbf3 100644 --- a/tests/test_resources/test_secrets/test_secret.py +++ b/tests/test_resources/test_secrets/test_secret.py @@ -84,7 +84,7 @@ class TestSecret(tests.test_resources.test_resource.TestResource): "secret": ["test_secret"] + provider_secrets, "cluster": [ "ondemand_aws_cluster", - "password_cluster", + "static_cpu_cluster", ], } MAXIMAL = { @@ -95,7 +95,6 @@ class TestSecret(tests.test_resources.test_resource.TestResource): "ondemand_k8s_cluster", "ondemand_aws_https_cluster_with_auth", "static_cpu_cluster", - "password_cluster", "multinode_cpu_cluster", "docker_cluster_pk_ssh_no_auth", "docker_cluster_pwd_ssh_no_auth",