Merge pull request #3093 from iterative/move-fs-utilities

Move fs utilities to fs.py from __init__.py

Author: efiop

dvc/analytics.py

@@ -16,7 +16,8 @@
 from dvc.lock import Lock, LockError
 from dvc.repo import Repo
 from dvc.scm import SCM
-from dvc.utils import env2bool, is_binary, makedirs
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
 
 
 logger = logging.getLogger(__name__)

dvc/remote/base.py

@@ -24,8 +24,8 @@
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
 from dvc.state import StateNoop
-from dvc.utils import makedirs, relpath, tmp_fname
-from dvc.utils.fs import move
+from dvc.utils import relpath, tmp_fname
+from dvc.utils.fs import move, makedirs
 from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)

dvc/remote/local.py

@@ -21,15 +21,13 @@
 from dvc.scheme import Schemes
 from dvc.scm.tree import is_working_tree
 from dvc.system import System
-from dvc.utils import copyfile
+from dvc.utils.fs import copyfile
 from dvc.utils import file_md5
-from dvc.utils import makedirs
 from dvc.utils import relpath
 from dvc.utils import tmp_fname
-from dvc.utils import walk_files
 from dvc.compat import fspath_py35
-from dvc.utils.fs import move
-from dvc.utils.fs import remove
+from dvc.utils.fs import move, makedirs, remove
+from dvc.utils.fs import walk_files
 
 logger = logging.getLogger(__name__)
 

dvc/repo/__init__.py

@@ -72,7 +72,7 @@ def __init__(self, root_dir=None):
         from dvc.repo.metrics import Metrics
         from dvc.scm.tree import WorkingTree
         from dvc.repo.tag import Tag
-        from dvc.utils import makedirs
+        from dvc.utils.fs import makedirs
 
         root_dir = self.find_root(root_dir)
 

dvc/utils/__init__.py

@@ -105,53 +105,6 @@ def dict_md5(d, exclude=()):
     return bytes_md5(byts)
 
 
-def copyfile(src, dest, no_progress_bar=False, name=None):
-    """Copy file with progress bar"""
-    from dvc.exceptions import DvcException
-    from dvc.progress import Tqdm
-    from dvc.system import System
-
-    src = fspath_py35(src)
-    dest = fspath_py35(dest)
-
-    name = name if name else os.path.basename(dest)
-    total = os.stat(src).st_size
-
-    if os.path.isdir(dest):
-        dest = os.path.join(dest, os.path.basename(src))
-
-    try:
-        System.reflink(src, dest)
-    except DvcException:
-        with Tqdm(
-            desc=name, disable=no_progress_bar, total=total, bytes=True
-        ) as pbar:
-            with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
-                while True:
-                    buf = fsrc.read(LOCAL_CHUNK_SIZE)
-                    if not buf:
-                        break
-                    fdest.write(buf)
-                    pbar.update(len(buf))
-
-
-def makedirs(path, exist_ok=False, mode=None):
-    path = fspath_py35(path)
-
-    if mode is None:
-        os.makedirs(path, exist_ok=exist_ok)
-        return
-
-    # utilize umask to set proper permissions since Python 3.7 the `mode`
-    # `makedirs` argument no longer affects the file permission bits of
-    # newly-created intermediate-level directories.
-    umask = os.umask(0o777 - mode)
-    try:
-        os.makedirs(path, exist_ok=exist_ok)
-    finally:
-        os.umask(umask)
-
-
 def _split(list_to_split, chunk_size):
     return [
         list_to_split[i : i + chunk_size]
@@ -278,12 +231,6 @@ def to_yaml_string(data):
     return stream.getvalue()
 
 
-def walk_files(directory):
-    for root, _, files in os.walk(fspath(directory)):
-        for f in files:
-            yield os.path.join(root, f)
-
-
 def colorize(message, color=None):
     """Returns a message in a specified color."""
     if not color:

dvc/utils/fs.py

@@ -18,6 +18,8 @@
 
 logger = logging.getLogger(__name__)
 
+LOCAL_CHUNK_SIZE = 2 ** 20  # 1 MB
+
 
 def fs_copy(src, dst):
     if os.path.isdir(src):
@@ -152,3 +154,56 @@ def normalize_path(path):
     parent = os.path.join(normalize_path(parent), "")
     child = normalize_path(child)
     return child != parent and child.startswith(parent)
+
+
+def makedirs(path, exist_ok=False, mode=None):
+    path = fspath_py35(path)
+
+    if mode is None:
+        os.makedirs(path, exist_ok=exist_ok)
+        return
+
+    # utilize umask to set proper permissions since Python 3.7 the `mode`
+    # `makedirs` argument no longer affects the file permission bits of
+    # newly-created intermediate-level directories.
+    umask = os.umask(0o777 - mode)
+    try:
+        os.makedirs(path, exist_ok=exist_ok)
+    finally:
+        os.umask(umask)
+
+
+def copyfile(src, dest, no_progress_bar=False, name=None):
+    """Copy file with progress bar"""
+    from dvc.exceptions import DvcException
+    from dvc.progress import Tqdm
+    from dvc.system import System
+
+    src = fspath_py35(src)
+    dest = fspath_py35(dest)
+
+    name = name if name else os.path.basename(dest)
+    total = os.stat(src).st_size
+
+    if os.path.isdir(dest):
+        dest = os.path.join(dest, os.path.basename(src))
+
+    try:
+        System.reflink(src, dest)
+    except DvcException:
+        with Tqdm(
+            desc=name, disable=no_progress_bar, total=total, bytes=True
+        ) as pbar:
+            with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
+                while True:
+                    buf = fsrc.read(LOCAL_CHUNK_SIZE)
+                    if not buf:
+                        break
+                    fdest.write(buf)
+                    pbar.update(len(buf))
+
+
+def walk_files(directory):
+    for root, _, files in os.walk(fspath(directory)):
+        for f in files:
+            yield os.path.join(root, f)

tests/dir_helpers.py

@@ -50,7 +50,7 @@
 import pytest
 from funcy.py3 import lmap, retry
 
-from dvc.utils import makedirs
+from dvc.utils.fs import makedirs
 from dvc.compat import fspath, fspath_py35
 
 

tests/func/test_checkout.py

@@ -18,7 +18,8 @@
 from dvc.stage import StageFileBadNameError
 from dvc.stage import StageFileDoesNotExistError
 from dvc.system import System
-from dvc.utils import relpath, walk_files
+from dvc.utils import relpath
+from dvc.utils.fs import walk_files
 from dvc.utils.stage import dump_stage_file
 from dvc.utils.stage import load_stage_file
 from tests.basic_env import TestDvc

tests/func/test_fs.py

@@ -0,0 +1,36 @@
+import os
+import stat
+
+import pytest
+
+from dvc.utils.fs import makedirs, copyfile
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
+def test_makedirs_permissions(tmp_dir):
+    dir_mode = 0o755
+    intermediate_dir = "тестовая-директория"
+    test_dir = os.path.join(intermediate_dir, "data")
+
+    assert not os.path.exists(intermediate_dir)
+
+    makedirs(test_dir, mode=dir_mode)
+
+    assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode
+    assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode
+
+
+def test_copyfile(tmp_dir):
+    src = "file1"
+    dest = "file2"
+    dest_dir = "testdir"
+
+    tmp_dir.gen(src, "file1contents")
+
+    os.mkdir(dest_dir)
+
+    copyfile(src, dest)
+    assert (tmp_dir / dest).read_text() == "file1contents"
+
+    copyfile(src, dest_dir)
+    assert (tmp_dir / dest_dir / src).read_text() == "file1contents"

tests/func/test_get.py

@@ -9,7 +9,7 @@
 from dvc.repo.get import GetDVCFileError, PathMissingError
 from dvc.repo import Repo
 from dvc.system import System
-from dvc.utils import makedirs
+from dvc.utils.fs import makedirs
 from dvc.compat import fspath
 from tests.utils import trees_equal