Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/folder-http-tests' into folder-h…
Browse files Browse the repository at this point in the history 
…ttp-tests

# Conflicts:
#	runhouse/servers/http/http_utils.py
#	tests/test_servers/test_http_server.py
  • Loading branch information
@jlewitt1
jlewitt1 committed Aug 6, 2024
2 parents 3d2d384 + 5b504d8 commit 2c6a0c8
Show file tree
Hide file tree
Showing 2 changed files with 235 additions and 85 deletions.
193 changes: 193 additions & 0 deletions runhouse/servers/http/http_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,3 +500,196 @@ def folder_mv(path: Path, folder_params: FolderParams):
shutil.move(str(path), str(dest_path))

return Response(output_type=OutputType.SUCCESS)


###########################
#### Folder Operations ####
###########################


def resolve_folder_path(path: str):
return (
None
if path is None
else Path(path).expanduser()
if path.startswith("~")
else Path(path).resolve()
)


def folder_mkdir(path: Path):
if not path.parent.is_dir():
raise ValueError(
f"Parent path {path.parent} does not exist or is not a directory"
)

path.mkdir(parents=True, exist_ok=True)

return Response(output_type=OutputType.SUCCESS)


def folder_get(path: Path, folder_params: FolderParams):
mode = folder_params.mode or "rb"
serialization = folder_params.serialization
binary_mode = "b" in mode

if not path.exists():
raise HTTPException(status_code=404, detail=f"Path {path} does not exist")

try:
with open(path, mode=mode) as f:
file_contents = f.read()
except FileNotFoundError:
raise HTTPException(status_code=404, detail=f"File {path} not found")

except PermissionError:
raise HTTPException(
status_code=403, detail=f"Permission denied for file in path {path}"
)

except Exception as e:
raise HTTPException(
status_code=500, detail=f"Error reading file {path}: {str(e)}"
)

if binary_mode and isinstance(file_contents, bytes):
file_contents = file_contents.decode()

output_type = OutputType.RESULT_SERIALIZED

return Response(
data=file_contents,
output_type=output_type,
serialization=serialization,
)


def folder_put(path: Path, folder_params: FolderParams):
overwrite = folder_params.overwrite
mode = folder_params.mode or "wb"
serialization = folder_params.serialization
contents = folder_params.contents

if not path.is_dir():
raise HTTPException(status_code=404, detail=f"Path {path} is not a directory")

path.mkdir(exist_ok=True)

if overwrite is False:
existing_files = {str(item.name) for item in path.iterdir()}
intersection = existing_files.intersection(set(contents.keys()))
if intersection:
raise HTTPException(
status_code=409,
detail=f"File(s) {intersection} already exist(s) at path: {path}",
)

for filename, file_obj in contents.items():
binary_mode = "b" in mode

if serialization:
file_obj = serialize_data(file_obj, serialization)

if binary_mode and not isinstance(file_obj, bytes):
file_obj = file_obj.encode()

file_path = path / filename
if not overwrite and file_path.exists():
raise HTTPException(
status_code=409, detail=f"File {file_path} already exists"
)

try:
with open(file_path, mode) as f:
f.write(file_obj)
except Exception as e:
HTTPException(status_code=500, detail=f"Failed to write file: {str(e)}")

return Response(output_type=OutputType.SUCCESS)


def folder_ls(path: Path, folder_params: FolderParams):
if not path.exists():
raise HTTPException(status_code=404, detail=f"Path {path} does not exist")

if not path.is_dir():
raise HTTPException(status_code=400, detail=f"Path {path} is not a directory")

files = (
list(path.rglob("*"))
if folder_params.recursive
else [item for item in path.iterdir()]
)
return Response(
data=files,
output_type=OutputType.RESULT_SERIALIZED,
serialization=None,
)


def folder_rm(path: Path, folder_params: FolderParams):
recursive: bool = folder_params.recursive
contents = folder_params.contents
if contents:
for content in contents:
content_path = path / content
if content_path.exists():
if content_path.is_file():
content_path.unlink()
elif content_path.is_dir() and recursive:
shutil.rmtree(content_path)
else:
raise HTTPException(
status_code=400,
detail=f"Path {content_path} is a directory and recursive is set to False",
)

return Response(output_type=OutputType.SUCCESS)

if not path.is_dir():
path.unlink()
return Response(output_type=OutputType.SUCCESS)

if recursive:
shutil.rmtree(path)
return Response(output_type=OutputType.SUCCESS)

items = list(path.iterdir())
if not items:
# Remove the empty directory
path.rmdir()
return Response(output_type=OutputType.SUCCESS)

# Remove file contents, but not the directory itself (since recursive not set to `True`)
for item in items:
if item.is_file():
item.unlink()
else:
raise HTTPException(
status_code=400,
detail=f"Folder {item} found in {path}, recursive is set to `False`",
)

return Response(output_type=OutputType.SUCCESS)


def folder_mv(path: Path, folder_params: FolderParams):
dest_path = resolve_folder_path(folder_params.dest_path)

if not path.exists():
raise HTTPException(
status_code=404, detail=f"The source path {path} does not exist"
)

if dest_path.exists():
raise HTTPException(
status_code=409, detail=f"The destination path {dest_path} already exists"
)

# Create the destination directory if it doesn't exist
dest_path.parent.mkdir(parents=True, exist_ok=True)

# Move the directory
shutil.move(str(path), str(dest_path))

return Response(output_type=OutputType.SUCCESS)
127 changes: 42 additions & 85 deletions tests/test_servers/test_http_server.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import json
import os
import tempfile
import uuid
from pathlib import Path

import pytest
Expand Down Expand Up @@ -107,70 +108,6 @@ def test_rename_object(self, http_client, cluster):
)
assert new_key in response.json().get("data")

@pytest.mark.level("local")
def test_folder_ls(self, http_client, cluster):
response = http_client.post(
"/folder",
json={"operation": "ls", "path": "~"},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

file_names: list = response.json().get("data")
base_names = [os.path.basename(f) for f in file_names]
assert ".rh" in base_names

@pytest.mark.level("local")
def test_folder_put_and_get(self, http_client, cluster):
response = http_client.post(
"/folder",
json={
"operation": "put",
"path": "~/.rh",
"contents": {"new_file.txt": "Hello, world!"},
},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

response = http_client.post(
"/folder",
json={"operation": "get", "path": "~/.rh/new_file.txt"},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

@pytest.mark.level("local")
def test_folder_mkdir(self, http_client, cluster):
response = http_client.post(
"/folder",
json={"operation": "mkdir", "path": "~/.rh/new-folder"},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

@pytest.mark.level("local")
def test_folder_rm(self, http_client, cluster):
# Delete the file
response = http_client.post(
"/folder",
json={
"operation": "rm",
"path": "~/.rh/new-folder",
"contents": ["new_file.txt"],
},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

# Delete the folder
response = http_client.post(
"/folder",
json={"operation": "rm", "path": "~/.rh/new-folder"},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

@pytest.mark.level("local")
def test_delete_obj(self, http_client, cluster):
# https://www.python-httpx.org/compatibility/#request-body-on-http-methods
Expand Down Expand Up @@ -281,33 +218,48 @@ def test_log_streaming_call(self, http_client, remote_log_streaming_func):
)

@pytest.mark.level("local")
def test_folder_put_and_get(self, http_client, cluster):
def test_folder_put_pickle_object(self, http_client, cluster):
file_name = str(uuid.uuid4())

raw_data = [1, 2, 3]
serialization = "pickle"
serialized_data = serialize_data(raw_data, serialization)

response = http_client.post(
"/folder/method/put",
json={
"path": "~/.rh",
"contents": {"new_file.txt": "Hello, world!"},
"contents": {f"{file_name}.pickle": serialized_data},
},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

response = http_client.post(
"/folder/method/get",
json={"path": "~/.rh/new_file.txt"},
json={"path": f"~/.rh/{file_name}.pickle"},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

resp_json = response.json()
serialization = resp_json["serialization"]

assert serialization == "pickle"
assert resp_json["output_type"] == "result_serialized"
assert deserialize_data(resp_json["data"], serialization) == "Hello, world!"

# TODO: User's responsibility to deserialize the response data?
assert deserialize_data(resp_json["data"], serialization) == raw_data

@pytest.mark.level("local")
def test_folder_mv(self, http_client, cluster):
def test_folder_put_and_mv_and_get(self, http_client, cluster):
response = http_client.post(
"/folder/method/put",
json={
"path": "~/.rh",
"contents": {"new_file.txt": "Hello, world!"},
},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

response = http_client.post(
"/folder/method/mv",
json={"path": "~/.rh/new_file.txt", "dest_path": "~/new_file.txt"},
Expand All @@ -322,40 +274,44 @@ def test_folder_mv(self, http_client, cluster):
)
assert response.status_code == 200

response = http_client.post(
"/folder/method/get",
json={"path": "~/.rh/new_file.txt"},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 404

@pytest.mark.level("local")
def test_folder_mkdir(self, http_client, cluster):
def test_folder_mkdir_and_rm(self, http_client, cluster):
response = http_client.post(
"/folder/method/mkdir",
json={"path": "~/.rh/new-folder"},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

@pytest.mark.level("local")
def test_folder_rm_and_ls(self, http_client, cluster):
# Delete the file
# Add a file to the new folder
response = http_client.post(
"/folder/method/rm",
"/folder/method/put",
json={
"path": "~/.rh/new-folder",
"contents": ["new_file.txt"],
"contents": {"new_file.txt": "Hello, world!"},
},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

# empty folder should still be there since recursive not explicitly set to `True`
# Delete specific contents from the folder, which will leave the folder empty but not delete it
response = http_client.post(
"/folder/method/ls",
json={"path": "~/.rh"},
"/folder/method/rm",
json={
"path": "~/.rh/new-folder",
"contents": ["new_file.txt"],
},
headers=rns_client.request_headers(cluster.rns_address),
)
assert response.status_code == 200

file_names: list = response.json().get("data")
base_names = [os.path.basename(f) for f in file_names]
assert "new-folder" in base_names

# Delete the now empty folder
response = http_client.post(
"/folder/method/rm",
Expand All @@ -364,6 +320,7 @@ def test_folder_rm_and_ls(self, http_client, cluster):
)
assert response.status_code == 200

# Confirm new folder does not exist
response = http_client.post(
"/folder/method/ls",
json={"path": "~/.rh"},
Expand Down

0 comments on commit 2c6a0c8

Please sign in to comment.