gs: adjust tests

Author: skshetry

dvc/remote/gs.py

@@ -176,7 +176,7 @@ def exists(self, path_info):
 
         eg: if `data/file.txt` exists, check for `data` should return True
         """
-        return self.isfile(path_info) or self.isdir(path_info / "")
+        return self.isfile(path_info) or self.isdir(path_info)
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=True):
         bucket = self.gs.bucket(to_info.bucket)

dvc/remote/s3.py

@@ -5,6 +5,7 @@
 import os
 import threading
 
+from botocore.exceptions import ClientError
 from funcy import cached_property, wrap_prop
 
 from dvc.config import Config
@@ -207,10 +208,26 @@ def _list_paths(self, path_info, max_items=None):
     def list_cache_paths(self):
         return self._list_paths(self.path_info)
 
+    def isfile(self, path_info):
+        if path_info.path.endswith("/"):
+            return False
+
+        try:
+            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
+        except ClientError as exc:
+            if exc.response["Error"]["Code"] != "404":
+                raise
+            return False
+
+        return True
+
     def exists(self, path_info):
-        dir_path = path_info / ""
-        fname = next(self._list_paths(path_info, max_items=1), "")
-        return path_info.path == fname or fname.startswith(dir_path.path)
+        """Check if the blob exists. If it does not exist,
+        it could be a part of a directory path.
+
+        eg: if `data/file.txt` exists, check for `data` should return True
+        """
+        return self.isfile(path_info) or self.isdir(path_info)
 
     def makedirs(self, path_info):
         # We need to support creating empty directories, which means
@@ -279,7 +296,7 @@ def _generate_download_url(self, path_info, expires=3600):
         )
 
     def walk_files(self, path_info, max_items=None):
-        for fname in self._list_paths(path_info, max_items):
+        for fname in self._list_paths(path_info / "", max_items):
             if fname.endswith("/"):
                 continue
 

tests/unit/remote/test_remote_dir.py

@@ -8,11 +8,13 @@
 remotes = [GCP, S3Mocked]
 
 FILE_WITH_CONTENTS = {
+    "data1.txt": "",
     "empty_dir/": "",
     "empty_file": "",
     "foo": "foo",
     "data/alice": "alice",
     "data/alpha": "alpha",
+    "data/subdir-file.txt": "subdir",
     "data/subdir/1": "1",
     "data/subdir/2": "2",
     "data/subdir/3": "3",
@@ -60,6 +62,7 @@ def test_exists(remote):
         (True, "data/subdir/1"),
         (False, "data/al"),
         (False, "foo/"),
+        (True, "data1.txt"),
     ]
 
     for expected, path in test_cases:
@@ -71,6 +74,7 @@ def test_walk_files(remote):
     files = [
         remote.path_info / "data/alice",
         remote.path_info / "data/alpha",
+        remote.path_info / "data/subdir-file.txt",
         remote.path_info / "data/subdir/1",
         remote.path_info / "data/subdir/2",
         remote.path_info / "data/subdir/3",
@@ -108,7 +112,7 @@ def test_makedirs(remote):
     assert remote.isdir(empty_dir)
 
 
-@pytest.mark.parametrize("remote", [GCP], indirect=True)
+@pytest.mark.parametrize("remote", [GCP, S3Mocked], indirect=True)
 def test_isfile(remote):
     test_cases = [
         (False, "empty_dir/"),