Merge remote-tracking branch 'origin/main' into rohin/log-format-impr…

…ovements

# Conflicts:
#	examples/airflow-torch-training/callables.py
#	runhouse/logger.py
#	runhouse/main.py
#	runhouse/resources/envs/conda_env.py
#	runhouse/resources/envs/env.py
#	runhouse/resources/folders/folder.py
#	runhouse/resources/folders/folder_factory.py
#	runhouse/resources/folders/gcs_folder.py
#	runhouse/resources/folders/s3_folder.py
#	runhouse/resources/functions/aws_lambda.py
#	runhouse/resources/functions/function.py
#	runhouse/resources/functions/function_factory.py
#	runhouse/resources/hardware/cluster.py
#	runhouse/resources/hardware/cluster_factory.py
#	runhouse/resources/hardware/on_demand_cluster.py
#	runhouse/resources/hardware/ray_utils.py
#	runhouse/resources/hardware/sagemaker/sagemaker_cluster.py
#	runhouse/resources/hardware/sky/command_runner.py
#	runhouse/resources/hardware/sky/subprocess_utils.py
#	runhouse/resources/hardware/sky_ssh_runner.py
#	runhouse/resources/module.py
#	runhouse/resources/packages/package.py
#	runhouse/resources/provenance.py
#	runhouse/resources/resource.py
#	runhouse/resources/secrets/provider_secrets/ssh_secret.py
#	runhouse/resources/secrets/secret.py
#	runhouse/resources/secrets/utils.py
#	runhouse/rns/defaults.py
#	runhouse/rns/login.py
#	runhouse/rns/rns_client.py
#	runhouse/rns/top_level_rns_fns.py
#	runhouse/servers/autostop_helper.py
#	runhouse/servers/caddy/config.py
#	runhouse/servers/cluster_servlet.py
#	runhouse/servers/env_servlet.py
#	runhouse/servers/http/auth.py
#	runhouse/servers/http/certs.py
#	runhouse/servers/http/http_client.py
#	runhouse/servers/http/http_server.py
#	runhouse/servers/http/http_utils.py
#	runhouse/servers/obj_store.py
#	runhouse/utils.py
#	tests/test_obj_store.py
#	tests/test_performance.py
#	tests/test_resources/test_modules/test_functions/test_function.py
#	tests/test_resources/test_modules/test_module.py
#	tests/test_servers/conftest.py

runhouse/logger.py

@@ -56,4 +56,4 @@ def get_logger(reinitialize: bool = False) -> logging.Logger:
         # Prevent log messages from propagating to the root logger
         logger.propagate = False
 
-    return logger
+    return logger

runhouse/resources/hardware/cluster.py

@@ -1143,6 +1143,9 @@ def rsync(
         # https://github.com/skypilot-org/skypilot/blob/v0.4.1/sky/backends/cloud_vm_ray_backend.py#L3094
         # This is an interesting policy... by default we're syncing to all nodes if the cluster is multinode.
         # If we need to change it to be greedier we can.
+        if up and not Path(source).expanduser().exists():
+            raise ValueError(f"Could not locate path to sync: {source}.")
+
         if up and (node == "all" or (len(self.ips) > 1 and not node)):
             for node in self.ips:
                 self.rsync(

runhouse/resources/packages/package.py

@@ -41,11 +41,11 @@ def path_to_sync_to_on_cluster(self) -> str:
         return (
             self._path_to_sync_to_on_cluster
             if self._path_to_sync_to_on_cluster
-            else f"~/{Path(self.local_path).name}"
+            else f"~/{Path(self.full_local_path_str()).name}"
         )
 
     def full_local_path_str(self) -> str:
-        return str(Path(self.local_path).expanduser())
+        return str(Path(self.local_path).expanduser().resolve())
 
     def __str__(self):
         return f"InstallTarget(local_path={self.local_path}, path_to_sync_to_on_cluster={self._path_to_sync_to_on_cluster})"
@@ -234,7 +234,7 @@ def _install(self, env: Union[str, "Env"] = None, cluster: "Cluster" = None):
         logger.info(f"Installing {str(self)} with method {self.install_method}.")
 
         if isinstance(self.install_target, InstallTarget):
-            if cluster:
+            if cluster and Path(self.install_target.local_path).expanduser().exists():
                 cluster.rsync(
                     source=str(self.install_target.local_path),
                     dest=str(self.install_target.path_to_sync_to_on_cluster),