Merge branch 'cluster-folder-methods' into test-cluster-folder-methods

runhouse/resources/folders/folder.py

@@ -131,7 +131,7 @@ def local_path(self):
         else:
             return None
 
-    def mv(self, system, path: Optional[str] = None) -> None:
+    def mv(self, system, path: Optional[str] = None, overwrite: bool = True) -> None:
         """Move the folder to a new filesystem or cluster.
 
         Example:
@@ -155,7 +155,7 @@ def mv(self, system, path: Optional[str] = None) -> None:
                 )
 
             # Create the destination directory if it doesn't exist
-            dest_path.parent.mkdir(parents=True, exist_ok=True)
+            dest_path.parent.mkdir(parents=True, exist_ok=overwrite)
 
             # Move the directory
             shutil.move(str(src_path), str(dest_path))

runhouse/resources/hardware/cluster.py

@@ -1835,7 +1835,7 @@ def _folder_mv(
         self,
         path: Union[str, Path],
         dest_path: Union[str, Path],
-        overwrite: bool = False,
+        overwrite: bool = True,
     ):
         return self.client.folder_mv(
             path=path, dest_path=dest_path, overwrite=overwrite

runhouse/servers/http/http_server.py

@@ -36,6 +36,7 @@
 from runhouse.servers.http.http_utils import (
     CallParams,
     DeleteObjectParams,
+    folder_exists,
     folder_get,
     folder_ls,
     folder_mkdir,
@@ -688,10 +689,10 @@ async def folder_put_cmd(request: Request, put_params: FolderPutParams):
             path = resolve_folder_path(put_params.path)
             return folder_put(
                 path,
+                contents=put_params.contents,
                 overwrite=put_params.overwrite,
                 mode=put_params.mode,
                 serialization=put_params.serialization,
-                contents=put_params.contents,
             )
 
         except Exception as e:
@@ -731,6 +732,19 @@ async def folder_mv_cmd(request: Request, mv_params: FolderMvParams):
                 e, traceback.format_exc(), from_http_server=True
             )
 
+    @staticmethod
+    @app.post("/folder/method/exists")
+    @validate_cluster_access
+    async def folder_exists_cmd(request: Request, folder_params: FolderParams):
+        try:
+            path = resolve_folder_path(folder_params.path)
+            return folder_exists(path=path)
+
+        except Exception as e:
+            return handle_exception_response(
+                e, traceback.format_exc(), from_http_server=True
+            )
+
     @staticmethod
     @app.post("/delete_object")
     @validate_cluster_access

runhouse/servers/http/http_utils.py

@@ -120,7 +120,7 @@ class FolderRmParams(FolderParams):
 
 class FolderMvParams(FolderParams):
     dest_path: str
-    overwrite: Optional[bool] = False
+    overwrite: Optional[bool] = True
 
     @validator("dest_path", pre=True, always=True)
     def convert_path_to_string(cls, v):
@@ -350,7 +350,7 @@ def folder_mkdir(path: Path):
     return Response(output_type=OutputType.SUCCESS)
 
 
-def folder_get(path: Path, mode: str = None, encoding: str = None):
+def folder_get(path: Path, encoding: str = None, mode: str = None):
     mode = mode or "rb"
     binary_mode = "b" in mode
 
@@ -386,10 +386,10 @@ def folder_get(path: Path, mode: str = None, encoding: str = None):
 
 def folder_put(
     path: Path,
+    contents: Dict[str, Any],
     overwrite: bool,
     mode: str = None,
     serialization: str = None,
-    contents: Dict[str, Any] = None,
 ):
     mode = mode or "wb"
 
@@ -399,8 +399,6 @@ def folder_put(
             detail="`contents` argument must be a dict mapping filenames to file-like objects",
         )
 
-    path.mkdir(parents=True, exist_ok=True)
-
     if overwrite is False:
         existing_files = {str(item.name) for item in path.iterdir()}
         intersection = existing_files.intersection(set(contents.keys()))
@@ -410,6 +408,8 @@ def folder_put(
                 detail=f"File(s) {intersection} already exist(s) at path: {path}",
             )
 
+    path.mkdir(parents=True, exist_ok=True)
+
     for filename, file_obj in contents.items():
         binary_mode = "b" in mode
 
@@ -523,3 +523,11 @@ def folder_mv(src_path: Path, dest_path: str, overwrite: bool):
     shutil.move(str(src_path), str(dest_path))
 
     return Response(output_type=OutputType.SUCCESS)
+
+
+def folder_exists(path: Path):
+    return Response(
+        data=path.exists() and path.is_dir(),
+        output_type=OutputType.RESULT_SERIALIZED,
+        serialization=None,
+    )

tests/conftest.py

@@ -77,7 +77,7 @@ class TestLevels(str, enum.Enum):
     MAXIMAL = "maximal"
 
 
-DEFAULT_LEVEL = TestLevels.LOCAL
+DEFAULT_LEVEL = TestLevels.UNIT
 
 TEST_LEVEL_HIERARCHY = {
     TestLevels.UNIT: 0,