Skip to content

Commit

Permalink
update docker folder fixture
Browse files Browse the repository at this point in the history
  • Loading branch information
jlewitt1 committed Aug 14, 2024
1 parent e560ebd commit 62b5dff
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 34 deletions.
35 changes: 18 additions & 17 deletions runhouse/resources/folders/folder.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,12 @@

from runhouse.logger import logger
from runhouse.resources.hardware import _current_cluster, _get_cluster_from, Cluster
from runhouse.resources.module import Module
from runhouse.resources.resource import Resource
from runhouse.rns.utils.api import generate_uuid, relative_file_path
from runhouse.utils import locate_working_dir


class Folder(Module):
class Folder(Resource):
RESOURCE_TYPE = "folder"
DEFAULT_FS = "file"
CLUSTER_FS = "ssh"
Expand All @@ -37,17 +36,21 @@ def __init__(
.. note::
To build a folder, please use the factory method :func:`folder`.
"""
super().__init__(name=name, dryrun=dryrun, system=system)
super().__init__(name=name, dryrun=dryrun)

# https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.gui.FileSelector.urlpath
# Note: no longer needed as part of previous fsspec usage, but still used by some s3 / gsutil commands
self._urlpath = None

self._system = _get_cluster_from(
system or _current_cluster(key="config"), dryrun=dryrun
)

# TODO [DG] Should we ever be allowing this to be None?
if path is None:
self._path = Folder.default_path(self.rns_address, system)
self._path = Folder.default_path(self.rns_address, self._system)
else:
if system != self.DEFAULT_FS:
if self._system != self.DEFAULT_FS:
self._path = path
else:
self._path = self._path_absolute_to_rh_workdir(path)
Expand Down Expand Up @@ -115,9 +118,17 @@ def path(self):
def path(self, path):
self._path = path

@property
def system(self):
return self._system

@system.setter
def system(self, new_system: Union[str, Cluster]):
self._system = _get_cluster_from(new_system)

@property
def _fs_str(self):
if isinstance(self.system, Resource): # if system is a cluster
if isinstance(self.system, Cluster):
if self.system.on_this_cluster():
return self.DEFAULT_FS
return self.CLUSTER_FS
Expand Down Expand Up @@ -217,17 +228,7 @@ def to(

# rsync the folder contents to the cluster's destination path
logger.debug(f"Syncing folder contents to cluster in path: {dest_path}")
self._to_cluster(system, path=dest_path)

# update the folder's system + path to the relative path on the cluster, since we'll return a
# new folder module which points to the cluster's file system
self.system = system
self.path = dest_path

# Note: setting `force_install` to ensure the module gets installed the cluster
# the folder's system may already be set to a cluster, which would skip the install
logger.debug("Sending folder module to cluster")
return super().to(system=system, force_install=True)
return self._to_cluster(system, path=dest_path)

path = str(
path or Folder.default_path(self.rns_address, system)
Expand Down
24 changes: 7 additions & 17 deletions tests/fixtures/folder_fixtures.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
from pathlib import Path

import pytest

import runhouse as rh
from runhouse.constants import TEST_ORG

from tests.conftest import init_args

Expand Down Expand Up @@ -37,25 +34,18 @@ def local_folder():

@pytest.fixture
def docker_cluster_folder(docker_cluster_pk_ssh_no_auth):
local_path = Path.cwd()
dest_path = "rh-folder"

args = {
"name": f"/{TEST_ORG}/test_docker_folder",
"path": local_path,
"name": "test_docker_folder",
"system": docker_cluster_pk_ssh_no_auth,
"path": "rh-folder",
}

# Create a local folder based on the current working dir, then send it to the docker cluster as a module
docker_folder = rh.folder(**args).to(
system=docker_cluster_pk_ssh_no_auth, path=dest_path
)
assert docker_folder.system == docker_cluster_pk_ssh_no_auth

init_args[id(docker_folder)] = args
docker_folder.put(
local_folder_docker = rh.folder(**args)
init_args[id(local_folder_docker)] = args
local_folder_docker.put(
{f"sample_file_{i}.txt": f"file{i}".encode() for i in range(3)}, overwrite=True
)
return docker_folder
return local_folder_docker


@pytest.fixture
Expand Down

0 comments on commit 62b5dff

Please sign in to comment.