Consolidate periodic loops into one function updating Den and updating autostop. #873
Conversation
This stack of pull requests is managed by Graphite. Learn more about stacking. |
rohinb2
force-pushed
the
06-04-Save_information_about_active_function_calls
branch
from
9cd3695
to
d53931e
Compare
rohinb2
force-pushed
the
06-04-Consolidate_periodic_loops_into_one_function_updating_Den_and_updating_autostop
branch
2 times, most recently
from
3cfdc6a
to
20d04a5
Compare
rohinb2
force-pushed
the
06-04-Save_information_about_active_function_calls
branch
from
d53931e
to
61bcf59
Compare
rohinb2
force-pushed
the
06-04-Consolidate_periodic_loops_into_one_function_updating_Den_and_updating_autostop
branch
from
20d04a5
to
baeb9bd
Compare
runhouse/servers/cluster_servlet.py
Outdated
| print("hello") | ||
| await asyncio.sleep(STATUS_CHECK_DELAY) | ||
| print("delayed") |
runhouse/rns/rns_client.py
Outdated
| @@ -659,3 +671,25 @@ def contents(self, name_or_path, full_paths): | |||
| return folder(name=name_or_path, path=folder_url).resources( | |||
| full_paths=full_paths | |||
| ) | |||
|
|
|||
| def send_status(self, status: ResourceStatusData, cluster_rns_address: str): | |||
There was a problem hiding this comment.
We need to catch all other errors so any failures here don't nuke the loop or cluster servlet.
There was a problem hiding this comment.
Guess async doesn't rly matter because its in its own thread, but can for consistency
There was a problem hiding this comment.
We need to catch all other errors so any failures here don't nuke the loop or cluster servlet.
The cluster servlet has everything in a try: except: so it'll catch it
| @@ -260,63 +246,45 @@ async def aclear_all_references_to_env_servlet_name(self, env_servlet_name: str) | |||
| # Cluster status functions | |||
| ############################################## | |||
|
|
|||
| async def asend_status_info_to_den(self): | |||
| async def aperiodic_status_check(self): | |||
| system_cpu_usage: float | ||
| system_memory_usage: Dict[str, Any] | ||
| system_disk_usage: Dict[str, Any] |
There was a problem hiding this comment.
Where does the GPU info go?
| def __init__(self): | ||
| self._last_activity = time.time() | ||
| self._last_register = None |
There was a problem hiding this comment.
I don't think we need the last_registered last_activity stuff anymore. We can just set the activity directly in SkyPilot, there's no reason to set it lazily like we did before.
There was a problem hiding this comment.
Nah currently we also do set_last_active_time_to_now in get_env_servlet_name_for_key. This happens quite often, we don't need to re-set it everytime, and the loop is always running, so we can just do it there.
runhouse/servers/cluster_servlet.py
Outdated
| if self.cluster_config.get("den_auth", False) and configs.token: | ||
| logger.debug("Creating send_status_info_to_den thread.") | ||
| if self.cluster_config.get("den_auth", False): | ||
| logger.info("Creating aperiodic_status_check thread.") |
There was a problem hiding this comment.
Ok this has to be on purpose
rohinb2
force-pushed
the
06-04-Consolidate_periodic_loops_into_one_function_updating_Den_and_updating_autostop
branch
from
baeb9bd
to
544fab0
Compare
rohinb2
force-pushed
the
06-04-Consolidate_periodic_loops_into_one_function_updating_Den_and_updating_autostop
branch
from
544fab0
to
65473ec
Compare
rohinb2
deleted the
06-04-Consolidate_periodic_loops_into_one_function_updating_Den_and_updating_autostop
branch
autostop.