Merge pull request #2913 from Suor/migrate-test-ignore

test: migrate test_ignore

Author: efiop

tests/dir_helpers.py

@@ -6,7 +6,7 @@
 from funcy.py3 import lmap, retry
 
 from dvc.utils import makedirs
-from dvc.utils.compat import basestring, is_py2, pathlib, fspath, fspath_py35
+from dvc.utils.compat import basestring, pathlib, fspath, fspath_py35, bytes
 
 
 __all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"]
@@ -55,13 +55,16 @@ def _gen(self, struct, prefix=None):
             path = (prefix or self) / name
 
             if isinstance(contents, dict):
-                self._gen(contents, prefix=path)
+                if not contents:
+                    makedirs(path, exist_ok=True)
+                else:
+                    self._gen(contents, prefix=path)
             else:
                 makedirs(path.parent, exist_ok=True)
-                if is_py2 and isinstance(contents, str):
+                if isinstance(contents, bytes):
                     path.write_bytes(contents)
                 else:
-                    path.write_text(contents)
+                    path.write_text(contents, encoding="utf-8")
 
     def dvc_gen(self, struct, text="", commit=None):
         paths = self.gen(struct, text)

tests/func/test_ignore.py

@@ -1,186 +1,121 @@
-import itertools
+# encoding: utf-8
+from __future__ import unicode_literals
 import os
 import shutil
-
 import pytest
 
 from dvc.exceptions import DvcIgnoreInCollectedDirError
-from dvc.ignore import DvcIgnore
-from dvc.ignore import DvcIgnoreDirs
-from dvc.ignore import DvcIgnoreFilter
-from dvc.ignore import DvcIgnorePatterns
+from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns
 from dvc.scm.tree import WorkingTree
-from dvc.utils.compat import cast_bytes
+from dvc.utils import walk_files
+from dvc.utils.compat import fspath
 from dvc.utils.fs import get_mtime_and_size
-from tests.basic_env import TestDvc
-from tests.utils import to_posixpath
-
 
-class TestDvcIgnore(TestDvc):
-    def setUp(self):
-        super(TestDvcIgnore, self).setUp()
-
-    def _get_all_paths(self):
-
-        paths = []
-        for root, dirs, files in self.dvc.tree.walk(
-            self.dvc.root_dir, dvcignore=self.dvc.dvcignore
-        ):
-            for dname in dirs:
-                paths.append(os.path.join(root, dname))
+from tests.utils import to_posixpath
 
-            for fname in files:
-                paths.append(os.path.join(root, fname))
 
-        return paths
+def test_ignore(tmp_dir, dvc, monkeypatch):
+    tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
 
-    def test_ignore_in_child_dir(self):
-        ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
-        with open(ignore_file, "w") as fobj:
-            fobj.write("data_dir/data")
+    assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
 
-        forbidden_path = os.path.join(self.dvc.root_dir, self.DATA)
-        all_paths = self._get_all_paths()
+    monkeypatch.chdir("dir")
+    assert _files_set(".", dvc.dvcignore) == {"./other"}
 
-        self.assertNotIn(forbidden_path, all_paths)
 
-    def test_ignore_in_child_dir_unicode(self):
-        ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
-        with open(ignore_file, "wb") as fobj:
-            fobj.write(cast_bytes(self.UNICODE, "utf-8"))
+def test_ignore_unicode(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"other": "text"}})
+    # Path() doesn't handle unicode paths in Windows/Python 2
+    # I don't know to debug it further, waiting till Python 2 EOL
+    with open("dir/тест", "wb") as fd:
+        fd.write("проверка".encode("utf-8"))
 
-        forbidden_path = os.path.join(self.dvc.root_dir, self.UNICODE)
-        all_paths = self._get_all_paths()
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
 
-        self.assertNotIn(forbidden_path, all_paths)
+    assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
 
-    def test_ignore_in_parent_dir(self):
-        ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
-        with open(ignore_file, "w") as fobj:
-            fobj.write("data_dir/data")
 
-        os.chdir(self.DATA_DIR)
+def test_rename_ignored_file(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
 
-        forbidden_path = os.path.join(self.dvc.root_dir, self.DATA)
-        all_paths = self._get_all_paths()
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
+    mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
 
-        self.assertNotIn(forbidden_path, all_paths)
+    shutil.move("dir/ignored", "dir/ignored_new")
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
 
+    assert new_mtime == mtime and new_size == size
 
-def test_metadata_unchanged_when_moving_ignored_file(dvc_repo, repo_dir):
-    new_data_path = repo_dir.DATA_SUB + "_new"
 
-    ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE)
-    repo_dir.create(
-        ignore_file,
-        "\n".join(
-            [to_posixpath(repo_dir.DATA_SUB), to_posixpath(new_data_path)]
-        ),
-    )
+def test_rename_file(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
+    mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
 
-    mtime_sig, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
+    shutil.move("dir/foo", "dir/foo_new")
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
 
-    shutil.move(repo_dir.DATA_SUB, new_data_path)
+    assert new_mtime != mtime and new_size == size
 
-    new_mtime_sig, new_size = get_mtime_and_size(
-        repo_dir.DATA_DIR, dvc_repo.dvcignore
-    )
 
-    assert new_mtime_sig == mtime_sig
-    assert new_size == size
+def test_remove_ignored_file(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
+    tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
 
+    mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
 
-def test_mtime_changed_when_moving_non_ignored_file(dvc_repo, repo_dir):
-    new_data_path = repo_dir.DATA_SUB + "_new"
-    mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
+    os.remove("dir/ignored")
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
 
-    shutil.move(repo_dir.DATA_SUB, new_data_path)
-    new_mtime, new_size = get_mtime_and_size(
-        repo_dir.DATA_DIR, dvc_repo.dvcignore
-    )
+    assert new_mtime == mtime and new_size == size
 
-    assert new_mtime != mtime
-    assert new_size == size
 
+def test_remove_file(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
+    mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
 
-def test_metadata_unchanged_on_ignored_file_deletion(dvc_repo, repo_dir):
-    ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE)
-    repo_dir.create(ignore_file, to_posixpath(repo_dir.DATA_SUB))
+    os.remove("dir/foo")
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
 
-    mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
+    assert new_mtime != mtime and new_size != size
 
-    os.remove(repo_dir.DATA_SUB)
-    new_mtime, new_size = get_mtime_and_size(
-        repo_dir.DATA_DIR, dvc_repo.dvcignore
-    )
 
-    assert new_mtime == mtime
-    assert new_size == size
+def test_dvcignore_in_out_dir(tmp_dir, dvc):
+    tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}})
 
+    with pytest.raises(DvcIgnoreInCollectedDirError):
+        dvc.add("dir")
 
-def test_metadata_changed_on_non_ignored_file_deletion(dvc_repo, repo_dir):
-    mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
 
-    os.remove(repo_dir.DATA_SUB)
-    new_mtime_sig, new_size = get_mtime_and_size(
-        repo_dir.DATA_DIR, dvc_repo.dvcignore
-    )
+# @efiop why do we need to parametrize this?
+@pytest.mark.parametrize("dname", ["dir", "dir/subdir"])
+def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
+    tmp_dir.gen({"dir": {"subdir": {}}})
 
-    assert new_mtime_sig != mtime
-    assert new_size != size
+    top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
+    top_ignore_file.write_text(os.path.basename(dname))
 
+    ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
+    ignore_file.write_text("foo")
 
-def test_should_raise_on_dvcignore_in_out_dir(dvc_repo, repo_dir):
-    ignore_file = os.path.join(repo_dir.DATA_DIR, DvcIgnore.DVCIGNORE_FILE)
-    repo_dir.create(ignore_file, "")
+    assert dvc.dvcignore.ignores == {
+        DvcIgnoreDirs([".git", ".hg", ".dvc"]),
+        DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)),
+    }
 
-    with pytest.raises(DvcIgnoreInCollectedDirError):
-        dvc_repo.add(repo_dir.DATA_DIR)
 
+def test_ignore_on_branch(tmp_dir, scm, dvc):
+    tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files")
 
-@pytest.mark.parametrize("dname", [TestDvc.DATA_DIR, TestDvc.DATA_SUB_DIR])
-def test_ignore_collecting_dvcignores(repo_dir, dname):
-    top_ignore_file = os.path.join(
-        repo_dir.root_dir, os.path.dirname(dname), DvcIgnore.DVCIGNORE_FILE
-    )
-    repo_dir.create(top_ignore_file, os.path.basename(dname))
+    scm.checkout("branch", create_new=True)
+    tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
 
-    ignore_file = os.path.join(
-        repo_dir.root_dir, dname, DvcIgnore.DVCIGNORE_FILE
-    )
-    repo_dir.create(ignore_file, repo_dir.FOO)
+    scm.checkout("master")
+    assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"}
 
-    assert DvcIgnoreFilter(
-        repo_dir.root_dir, WorkingTree(repo_dir.root_dir)
-    ).ignores == {
-        DvcIgnoreDirs([".git", ".hg", ".dvc"]),
-        DvcIgnorePatterns(top_ignore_file, WorkingTree(repo_dir.root_dir)),
-    }
+    dvc.tree = scm.get_tree("branch")
+    assert _files_set(".", dvc.dvcignore) == {"./bar"}
 
 
-def test_ignore_on_branch(git, dvc_repo, repo_dir):
-    dvc_repo.add(repo_dir.DATA_DIR)
-    dvc_repo.scm.commit("add data dir")
-
-    branch_name = "branch_one"
-    dvc_repo.scm.checkout(branch_name, create_new=True)
-
-    repo_dir.create(DvcIgnore.DVCIGNORE_FILE, to_posixpath(repo_dir.DATA_SUB))
-    dvc_repo.scm.add([DvcIgnore.DVCIGNORE_FILE])
-    git.index.commit("add ignore")
-
-    dvc_repo.scm.checkout("master")
-
-    git_tree = dvc_repo.scm.get_tree(branch_name)
-    branch_data_files = set(
-        itertools.chain.from_iterable(
-            [
-                files
-                for _, _, files in dvc_repo.tree.walk(
-                    repo_dir.DATA_DIR,
-                    dvcignore=DvcIgnoreFilter(repo_dir.root_dir, git_tree),
-                )
-            ]
-        )
-    )
-    assert branch_data_files == {"data"}
+def _files_set(root, dvcignore):
+    return {to_posixpath(f) for f in walk_files(root, dvcignore)}