Merge pull request #3135 from rxxg/master

Perf improvement for file copies

Author: efiop

dvc/system.py

@@ -1,14 +1,31 @@
 import errno
 import logging
 import os
+import platform
 import shutil
 
 from dvc.compat import fspath
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
+if platform.system() == "Windows":
+    try:
+        import speedcopy
+
+        speedcopy.patch_copyfile()
+    except ImportError:
+        pass
+else:
+    import sys
+
+    if sys.version_info < (3, 8):
+        try:
+            # Importing the module monkey-patches shutil.copyfile
+            import pyfastcopy  # noqa: F401
+        except ImportError:
+            pass
+
 
 class System(object):
     @staticmethod
@@ -90,8 +107,6 @@ def _reflink_linux(src, dst):
 
     @staticmethod
     def reflink(source, link_name):
-        import platform
-
         source, link_name = fspath(source), fspath(link_name)
 
         system = platform.system()

setup.py

@@ -75,6 +75,8 @@ def run(self):
     "win-unicode-console>=0.5; sys_platform == 'win32'",
     "pywin32>=225; sys_platform == 'win32'",
     "networkx>=2.1,<2.4",
+    "speedcopy>=2.0.1",
+    "pyfastcopy>=1.0.3",
     "flatten-dict>=0.2.0",
     "texttable>=0.5.2",
 ]