test: refactor tmp dir helper fixtures

- `tmp_dir` descends from Path
- convenient template methods to generate files and dirs, add them to
dvc and git and commit to git
- independent of old basic_env structures
- can go in any order
- modular: start with empty `tmp_dir`, initialize git with `scm`, dvc
repo with `dvc` add "repo template" with `repo_template`, ...

dvc/utils/compat.py

@@ -75,6 +75,8 @@ def _makedirs(name, mode=0o777, exist_ok=False):
     """Source: https://github.com/python/cpython/blob/
         3ce3dea60646d8a5a1c952469a2eb65f937875b3/Lib/os.py#L196-L226
     """
+    name = fspath_py35(name)
+
     head, tail = os.path.split(name)
     if not tail:
         head, tail = os.path.split(head)

tests/conftest.py

@@ -7,13 +7,13 @@
 from git import Repo
 from git.exc import GitCommandNotFound
 
-from .basic_env import TestDirFixture
-from .basic_env import TestDvcGitFixture
-from .basic_env import TestGitFixture
 from dvc.remote.config import RemoteConfig
 from dvc.remote.ssh.connection import SSHConnection
 from dvc.repo import Repo as DvcRepo
 from dvc.utils.compat import cast_bytes_py2
+from .basic_env import TestDirFixture, TestDvcGitFixture, TestGitFixture
+from .dir_helpers import *  # noqa
+
 
 # Prevent updater and analytics from running their processes
 os.environ[cast_bytes_py2("DVC_TEST")] = cast_bytes_py2("true")

tests/dir_helpers.py

@@ -0,0 +1,223 @@
+import os
+import pytest
+
+from funcy import lmap, retry
+
+from dvc.utils.compat import pathlib, fspath, makedirs, basestring
+
+
+__all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"]
+REPO_TEMPLATE = {
+    "foo": "foo",
+    "bar": "bar",
+    "dir": {
+        "data": "dir/data text",
+        "subdir": {"subdata": "dir/subdir/subdata text"},
+    },
+}
+
+
+class TmpDir(pathlib.Path):
+    def __new__(cls, *args, **kwargs):
+        if cls is TmpDir:
+            cls = WindowsTmpDir if os.name == "nt" else PosixTmpDir
+        self = cls._from_parts(args, init=False)
+        if not self._flavour.is_supported:
+            raise NotImplementedError(
+                "cannot instantiate %r on your system" % (cls.__name__,)
+            )
+        self._init()
+        return self
+
+    def _require(self, name):
+        if not hasattr(self, name):
+            raise TypeError(
+                "Can't use {name} for this temporary dir. "
+                'Did you forget to use "{name}" fixture?'.format(name=name)
+            )
+
+    def gen(self, struct, text=""):
+        if isinstance(struct, str):
+            struct = {struct: text}
+
+        return self._gen(struct)
+
+    def _gen(self, struct, prefix=None):
+        paths = []
+
+        for name, contents in struct.items():
+            path = (prefix or self) / name
+
+            if isinstance(contents, dict):
+                paths.extend(self._gen(contents, prefix=path))
+            else:
+                makedirs(path.parent, exist_ok=True)
+                path.write_text(contents)
+                paths.append(path.relative_to(self))
+
+        return paths
+
+    def dvc_gen(self, struct, text="", commit=None):
+        paths = self.gen(struct, text)
+        return self.dvc_add(paths, commit=commit)
+
+    def scm_gen(self, struct, text="", commit=None):
+        paths = self.gen(struct, text)
+        return self.scm_add(paths, commit=commit)
+
+    def dvc_add(self, filenames, commit=None):
+        self._require("dvc")
+        filenames = _coerce_filenames(filenames)
+
+        stages = self.dvc.add(filenames)
+        if commit:
+            stage_paths = [s.path for s in stages]
+            self.scm_add(stage_paths, commit=commit)
+
+        return stages
+
+    def scm_add(self, filenames, commit=None):
+        self._require("scm")
+        filenames = _coerce_filenames(filenames)
+
+        self.scm.add(filenames)
+        if commit:
+            self.scm.commit(commit)
+
+    # Introspection mehtods
+    def list(self):
+        return [p.name for p in self.iterdir()]
+
+
+def _coerce_filenames(filenames):
+    if isinstance(filenames, (basestring, pathlib.PurePath)):
+        filenames = [filenames]
+    return lmap(fspath, filenames)
+
+
+class WindowsTmpDir(TmpDir, pathlib.PureWindowsPath):
+    pass
+
+
+class PosixTmpDir(TmpDir, pathlib.PurePosixPath):
+    pass
+
+
+@pytest.fixture
+def tmp_dir(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    return TmpDir(tmp_path)
+
+
+@pytest.fixture
+def scm(tmp_dir, request):
+    # Use dvc.scm if available
+    if "dvc" in request.fixturenames:
+        dvc = request.getfixturevalue("dvc")
+        tmp_dir.scm = dvc.scm
+        yield dvc.scm
+
+    else:
+        from dvc.scm.git import Git
+
+        _git_init()
+        try:
+            scm = tmp_dir.scm = Git(fspath(tmp_dir))
+            yield scm
+        finally:
+            scm.close()
+
+
+def _git_init():
+    from git import Repo
+    from git.exc import GitCommandNotFound
+
+    # NOTE: handles EAGAIN error on BSD systems (osx in our case).
+    # Otherwise when running tests you might get this exception:
+    #
+    #    GitCommandNotFound: Cmd('git') not found due to:
+    #        OSError('[Errno 35] Resource temporarily unavailable')
+    git = retry(5, GitCommandNotFound)(Repo.init)()
+    git.close()
+
+
+@pytest.fixture
+def dvc(tmp_dir, request):
+    from dvc.repo import Repo
+
+    if "scm" in request.fixturenames:
+        if not hasattr(tmp_dir, "scm"):
+            _git_init()
+
+        dvc = Repo.init(fspath(tmp_dir))
+        dvc.scm.commit("init dvc")
+    else:
+        dvc = Repo.init(fspath(tmp_dir), no_scm=True)
+
+    try:
+        tmp_dir.dvc = dvc
+        yield dvc
+    finally:
+        dvc.close()
+
+
+@pytest.fixture
+def repo_template(tmp_dir):
+    tmp_dir.gen(REPO_TEMPLATE)
+
+
+@pytest.fixture
+def run_copy(tmp_dir, dvc, request):
+    tmp_dir.gen(
+        "copy.py",
+        "import sys, shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])",
+    )
+
+    # Do we need this?
+    if "scm" in request.fixturenames:
+        request.getfixturevalue("scm")
+        tmp_dir.git_add("copy.py", commit="add copy.py")
+
+    def run_copy(src, dst, **run_kwargs):
+        return dvc.run(
+            cmd="python copy.py {} {}".format(src, dst),
+            outs=[dst],
+            deps=[src, "copy.py"],
+            **run_kwargs,
+        )
+
+    return run_copy
+
+
+@pytest.fixture
+def erepo_dir(tmp_path_factory, monkeypatch):
+    from dvc.repo import Repo
+    from dvc.remote.config import RemoteConfig
+
+    path = TmpDir(tmp_path_factory.mktemp("erepo"))
+    path.gen(REPO_TEMPLATE)
+
+    # Chdir for git and dvc to work locally
+    monkeypatch.chdir(path)
+
+    _git_init()
+    path.dvc = Repo.init()
+    path.scm = path.dvc.scm
+
+    path.dvc_add(["foo", "bar", "dir"], commit="init repo")
+
+    rconfig = RemoteConfig(path.dvc.config)
+    rconfig.add("upstream", path.dvc.cache.local.cache_dir, default=True)
+    path.scm_add([path.dvc.config.config_file], commit="add remote")
+
+    path.dvc_gen("version", "master", commit="master")
+
+    path.scm.checkout("branch", create_new=True)
+    path.dvc_gen("version", "branch")
+    path.scm_add([".gitignore", "version.dvc"], commit="branch")
+
+    path.scm.checkout("master")
+    path.dvc.close()
+    monkeypatch.undo()  # Undo chdir
+
+    return path

tests/func/test_add.py

@@ -59,7 +59,7 @@ def test_unicode(self):
         self.assertTrue(os.path.isfile(stage.path))
 
 
-class TestAddUnSupportedFile(TestDvc):
+class TestAddUnupportedFile(TestDvc):
     def test(self):
         with self.assertRaises(DvcException):
             self.dvc.add("unsupported://unsupported")
@@ -132,15 +132,11 @@ def test(self):
         self.assertEqual(os.path.abspath("directory.dvc"), stage.path)
 
 
-def test_add_tracked_file(git, dvc_repo, repo_dir):
-    fname = "tracked_file"
-    repo_dir.create(fname, "tracked file contents")
-
-    dvc_repo.scm.add([fname])
-    dvc_repo.scm.commit("add {}".format(fname))
+def test_add_tracked_file(tmp_dir, scm, dvc):
+    tmp_dir.scm_gen("tracked_file", "...", commit="add tracked file")
 
     with pytest.raises(OutputAlreadyTrackedError):
-        dvc_repo.add(fname)
+        dvc.add("tracked_file")
 
 
 class TestAddDirWithExistingCache(TestDvc):
@@ -474,23 +470,18 @@ def test(self):
         self.assertFalse(os.path.exists("foo.dvc"))
 
 
-def test_should_cleanup_after_failed_add(git, dvc_repo, repo_dir):
-    stages = dvc_repo.add(repo_dir.FOO)
-    assert len(stages) == 1
-
-    foo_stage_file = repo_dir.FOO + Stage.STAGE_FILE_SUFFIX
-
-    # corrupt stage file
-    repo_dir.create(foo_stage_file, "this will break yaml structure")
+def test_should_cleanup_after_failed_add(tmp_dir, scm, dvc, repo_template):
+    # Add and corrupt a stage file
+    dvc.add("foo")
+    tmp_dir.gen("foo.dvc", "- broken\nyaml")
 
     with pytest.raises(StageFileCorruptedError):
-        dvc_repo.add(repo_dir.BAR)
+        dvc.add("bar")
 
-    bar_stage_file = repo_dir.BAR + Stage.STAGE_FILE_SUFFIX
-    assert not os.path.exists(bar_stage_file)
+    assert not os.path.exists("bar.dvc")
 
     gitignore_content = get_gitignore_content()
-    assert "/" + repo_dir.BAR not in gitignore_content
+    assert "/bar" not in gitignore_content
 
 
 class TestShouldNotTrackGitInternalFiles(TestDvc):
@@ -634,7 +625,7 @@ def test_should_protect_on_repeated_add(link, dvc_repo, repo_dir):
     assert not os.access(repo_dir.FOO, os.W_OK)
 
 
-def test_escape_gitignore_entries(git, dvc_repo, repo_dir):
+def test_escape_gitignore_entries(tmp_dir, scm, dvc):
     fname = "file!with*weird#naming_[1].t?t"
     ignored_fname = r"/file\!with\*weird\#naming_\[1\].t\?t"
 
@@ -644,8 +635,5 @@ def test_escape_gitignore_entries(git, dvc_repo, repo_dir):
         fname = "file!with_weird#naming_[1].txt"
         ignored_fname = r"/file\!with_weird\#naming_\[1\].txt"
 
-    os.rename(repo_dir.FOO, fname)
-
-    dvc_repo.add(fname)
-
+    tmp_dir.dvc_gen(fname, "...")
     assert ignored_fname in get_gitignore_content()