Skip to content

Strip protocol in arrowfs.ls and add tests for calling arrowfs with protocol #1099

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martindurant merged 1 commit into from
Nov 7, 2022
Merged

Strip protocol in arrowfs.ls and add tests for calling arrowfs with protocol #1099

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions fsspec/implementations/arrow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def _strip_protocol(cls, path):
return path

def ls(self, path, detail=False, **kwargs):
path = self._strip_protocol(path)
from pyarrow.fs import FileSelector

entries = [
Expand Down
17 changes: 10 additions & 7 deletions fsspec/implementations/tests/test_arrow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ def fs():
return ArrowFSWrapper(fs)


@pytest.fixture(scope="function")
def remote_dir(fs):
@pytest.fixture(scope="function", params=[False, True])
def remote_dir(fs, request):
directory = secrets.token_hex(16)
fs.makedirs(directory)
yield directory
yield ("hdfs://" if request.param else "") + directory
fs.rm(directory, recursive=True)


Expand All @@ -30,18 +30,19 @@ def strip_keys(original_entry):

def test_info(fs, remote_dir):
fs.touch(remote_dir + "/a.txt")
remote_dir_strip_protocol = fs._strip_protocol(remote_dir)
details = fs.info(remote_dir + "/a.txt")
assert details["type"] == "file"
assert details["name"] == remote_dir + "/a.txt"
assert details["name"] == remote_dir_strip_protocol + "/a.txt"
assert details["size"] == 0

fs.mkdir(remote_dir + "/dir")
details = fs.info(remote_dir + "/dir")
assert details["type"] == "directory"
assert details["name"] == remote_dir + "/dir"
assert details["name"] == remote_dir_strip_protocol + "/dir"

details = fs.info(remote_dir + "/dir/")
assert details["name"] == remote_dir + "/dir/"
assert details["name"] == remote_dir_strip_protocol + "/dir/"


def test_move(fs, remote_dir):
Expand Down Expand Up @@ -114,12 +115,14 @@ def test_rm(fs, remote_dir):


def test_ls(fs, remote_dir):
remote_dir_strip_protocol = fs._strip_protocol(remote_dir)
fs.mkdir(remote_dir + "dir/")
files = set()
for no in range(8):
file = remote_dir + f"dir/test_{no}"
# we also want to make sure `fs.touch` works with protocol
fs.touch(file)
files.add(file)
files.add(remote_dir_strip_protocol + f"dir/test_{no}")

assert set(fs.ls(remote_dir + "dir/")) == files

Expand Down