Merge branch 'master' into refactor-remote-azure

Author: efiop

dvc/remote/s3.py

@@ -194,11 +194,7 @@ def _list_objects(self, path_info, max_items=None):
         }
         paginator = self.s3.get_paginator(self.list_objects_api)
         for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", None)
-            if not contents:
-                continue
-            for item in contents:
-                yield item
+            yield from page.get("Contents", ())
 
     def _list_paths(self, path_info, max_items=None):
         return (

dvc/remote/ssh/__init__.py

@@ -260,8 +260,7 @@ def open(self, path_info, mode="r", encoding=None):
     def list_cache_paths(self):
         with self.ssh(self.path_info) as ssh:
             # If we simply return an iterator then with above closes instantly
-            for path in ssh.walk_files(self.path_info.path):
-                yield path
+            yield from ssh.walk_files(self.path_info.path)
 
     def walk_files(self, path_info):
         with self.ssh(path_info) as ssh:

dvc/remote/ssh/connection.py

@@ -139,8 +139,7 @@ def walk(self, directory, topdown=True):
 
         for dname in dirs:
             newpath = posixpath.join(directory, dname)
-            for entry in self.walk(newpath, topdown=topdown):
-                yield entry
+            yield from self.walk(newpath, topdown=topdown)
 
         if not topdown:
             yield directory, dirs, nondirs

dvc/scm/git/tree.py

@@ -128,8 +128,7 @@ def _walk(self, tree, topdown=True):
             yield os.path.normpath(tree.abspath), dirs, nondirs
 
         for i in dirs:
-            for x in self._walk(tree[i], topdown=True):
-                yield x
+            yield from self._walk(tree[i], topdown=topdown)
 
         if not topdown:
             yield os.path.normpath(tree.abspath), dirs, nondirs
@@ -146,5 +145,4 @@ def walk(self, top, topdown=True):
         if tree is None:
             raise IOError(errno.ENOENT, "No such file")
 
-        for x in self._walk(tree, topdown):
-            yield x
+        yield from self._walk(tree, topdown=topdown)

tests/dir_helpers.py

@@ -139,6 +139,7 @@ def scm_add(self, filenames, commit=None):
         if commit:
             self.scm.commit(commit)
 
+    # contexts
     @contextmanager
     def chdir(self):
         old = os.getcwd()
@@ -148,6 +149,16 @@ def chdir(self):
         finally:
             os.chdir(old)
 
+    @contextmanager
+    def branch(self, name, new=False):
+        self._require("scm")
+        old = self.scm.active_branch()
+        try:
+            self.scm.checkout(name, create_new=new)
+            yield
+        finally:
+            self.scm.checkout(old)
+
     # Introspection methods
     def list(self):
         return [p.name for p in self.iterdir()]
@@ -265,21 +276,12 @@ def erepo_dir(tmp_path_factory, monkeypatch):
         _git_init()
         path.dvc = Repo.init()
         path.scm = path.dvc.scm
-        path.dvc_gen(REPO_TEMPLATE, commit="init repo")
+        path.scm.commit("init dvc")
 
         rconfig = RemoteConfig(path.dvc.config)
         rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True)
         path.scm_add([path.dvc.config.config_file], commit="add remote")
 
-        path.dvc_gen("version", "master")
-        path.scm_add([".gitignore", "version.dvc"], commit="master")
-
-        path.scm.checkout("branch", create_new=True)
-        (path / "version").unlink()  # For mac ???
-        path.dvc_gen("version", "branch")
-        path.scm_add([".gitignore", "version.dvc"], commit="branch")
-
-        path.scm.checkout("master")
         path.dvc.close()
 
     return path

tests/func/test_api.py

@@ -30,17 +30,19 @@ def run_dvc(*argv):
 
 
 @pytest.mark.parametrize("remote_url", remote_params, indirect=True)
-def test_get_url(remote_url, tmp_dir, dvc, repo_template):
+def test_get_url(tmp_dir, dvc, remote_url):
     run_dvc("remote", "add", "-d", "upstream", remote_url)
-    dvc.add("foo")
+    tmp_dir.dvc_gen("foo", "foo")
 
     expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
     assert api.get_url("foo") == expected_url
 
 
 @pytest.mark.parametrize("remote_url", remote_params, indirect=True)
-def test_get_url_external(remote_url, erepo_dir):
+def test_get_url_external(erepo_dir, remote_url):
     _set_remote_url_and_commit(erepo_dir.dvc, remote_url)
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo", commit="add foo")
 
     # Using file url to force clone to tmp repo
     repo_url = "file://{}".format(erepo_dir)
@@ -63,11 +65,15 @@ def test_open(remote_url, tmp_dir, dvc):
 
 @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)
 def test_open_external(remote_url, erepo_dir):
-    erepo_dir.scm.checkout("branch")
-    _set_remote_url_and_commit(erepo_dir.dvc, remote_url)
-    erepo_dir.scm.checkout("master")
     _set_remote_url_and_commit(erepo_dir.dvc, remote_url)
 
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("version", "master", commit="add version")
+
+        with erepo_dir.branch("branch", new="True"):
+            # NOTE: need file to be other size for Mac
+            erepo_dir.dvc_gen("version", "branchver", commit="add version")
+
     erepo_dir.dvc.push(all_branches=True)
 
     # Remove cache to force download
@@ -78,7 +84,7 @@ def test_open_external(remote_url, erepo_dir):
     with api.open("version", repo=repo_url) as fd:
         assert fd.read() == "master"
 
-    assert api.read("version", repo=repo_url, rev="branch") == "branch"
+    assert api.read("version", repo=repo_url, rev="branch") == "branchver"
 
 
 @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True)

tests/func/test_data_cloud.py

@@ -33,11 +33,11 @@
 from tests.remotes import (
     _should_test_aws,
     _should_test_gcp,
-    _should_test_gdrive,
     _should_test_hdfs,
     _should_test_oss,
     _should_test_ssh,
     Azure,
+    GDrive,
     TEST_CONFIG,
     TEST_SECTION,
     TEST_GCP_CREDS_FILE,
@@ -46,7 +46,6 @@
     TEST_REMOTE,
     get_aws_url,
     get_gcp_url,
-    get_gdrive_url,
     get_hdfs_url,
     get_local_url,
     get_oss_url,
@@ -208,7 +207,7 @@ def _get_cloud_class(self):
 
 class TestRemoteGDrive(TestDataCloudBase):
     def _should_test(self):
-        return _should_test_gdrive()
+        return GDrive.should_test()
 
     def _setup_cloud(self):
         self._ensure_should_run()
@@ -229,7 +228,7 @@ def _setup_cloud(self):
         self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
 
     def _get_url(self):
-        return get_gdrive_url()
+        return GDrive.get_url()
 
     def _get_cloud_class(self):
         return RemoteGDrive
@@ -483,10 +482,10 @@ def _test(self):
 
 class TestRemoteGDriveCLI(TestDataCloudCLIBase):
     def _should_test(self):
-        return _should_test_gdrive()
+        return GDrive.should_test()
 
     def _test(self):
-        url = get_gdrive_url()
+        url = GDrive.get_url()
 
         self.main(["remote", "add", TEST_REMOTE, url])
         self.main(

tests/func/test_get.py

@@ -77,13 +77,10 @@ def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
 
 
 def test_get_repo_rev(tmp_dir, erepo_dir):
-    with erepo_dir.chdir():
-        erepo_dir.scm.checkout("new_branch", create_new=True)
+    with erepo_dir.chdir(), erepo_dir.branch("branch", new=True):
         erepo_dir.dvc_gen("file", "contents", commit="create file on branch")
-        erepo_dir.scm.checkout("master")
-
-    Repo.get(fspath(erepo_dir), "file", "file_imported", rev="new_branch")
 
+    Repo.get(fspath(erepo_dir), "file", "file_imported", rev="branch")
     assert (tmp_dir / "file_imported").read_text() == "contents"
 
 
@@ -159,11 +156,10 @@ def test_get_to_dir(tmp_dir, erepo_dir, dname):
 
 def test_get_from_non_dvc_master(tmp_dir, erepo_dir, caplog):
     with erepo_dir.chdir():
-        erepo_dir.scm.checkout("new_branch", create_new=True)
-        erepo_dir.scm_gen(
-            {"some_file": "some_contents"}, commit="create some file"
-        )
-        erepo_dir.scm.checkout("master")
+        with erepo_dir.branch("branch", new=True):
+            erepo_dir.scm_gen(
+                {"some_file": "some_contents"}, commit="create some file"
+            )
 
         erepo_dir.dvc.scm.repo.index.remove([".dvc"], r=True)
         erepo_dir.dvc.scm.commit("remove .dvc")
@@ -175,7 +171,7 @@ def test_get_from_non_dvc_master(tmp_dir, erepo_dir, caplog):
     caplog.clear()
     dst = "file_imported"
     with caplog.at_level(logging.INFO, logger="dvc"):
-        Repo.get(fspath(erepo_dir), "some_file", out=dst, rev="new_branch")
+        Repo.get(fspath(erepo_dir), "some_file", out=dst, rev="branch")
 
     assert caplog.text == ""
     assert (tmp_dir / dst).read_text() == "some_contents"

tests/func/test_ignore.py

@@ -107,10 +107,9 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
 def test_ignore_on_branch(tmp_dir, scm, dvc):
     tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files")
 
-    scm.checkout("branch", create_new=True)
-    tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
+    with tmp_dir.branch("branch", new=True):
+        tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
 
-    scm.checkout("master")
     assert _files_set(".", dvc.tree) == {"./foo", "./bar"}
 
     dvc.tree = scm.get_tree("branch")

tests/func/test_import.py

@@ -90,12 +90,10 @@ def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm):
 
 
 def test_import_rev(tmp_dir, scm, dvc, erepo_dir):
-    with erepo_dir.chdir():
-        erepo_dir.scm.checkout("new_branch", create_new=True)
+    with erepo_dir.chdir(), erepo_dir.branch("branch", new=True):
         erepo_dir.dvc_gen("foo", "foo content", commit="create foo on branch")
-        erepo_dir.scm.checkout("master")
 
-    dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="new_branch")
+    dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="branch")
 
     assert (tmp_dir / "foo_imported").read_text() == "foo content"
     assert scm.repo.git.check_ignore("foo_imported")
@@ -188,12 +186,13 @@ def test_import_to_dir(dname, tmp_dir, dvc, erepo_dir):
 def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("foo", "master content", commit="create foo")
-        erepo_dir.scm.checkout("new_branch", create_new=True)
-        erepo_dir.dvc_gen("foo", "branch content", commit="modify foo")
 
-    stage = dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="new_branch")
+        with erepo_dir.branch("branch", new=True):
+            erepo_dir.dvc_gen("foo", "branch content", commit="modify foo")
+
+    stage = dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="branch")
     tmp_dir.scm_add([stage.relpath], commit="imported branch")
-    dvc.scm.tag("ref-to-branch")
+    scm.tag("ref-to-branch")
 
     # Overwrite via import
     dvc.imp(fspath(erepo_dir), "foo", "foo_imported", rev="master")