Backport new readonly removal features from python

- Backport python/cpython#10320 (not yet merged) for better cleanup on
  `shutil.rmtree` failure (using `os.chflags`)
- Add compatibility shims for `PermissionError` and `IsADirectory`
  exceptions
- General cleanup
- Fixes #38

Signed-off-by: Dan Ryan <dan@danryan.co>

README.rst

@@ -113,11 +113,14 @@ Shims are provided for full API compatibility from python 2.7 through 3.7 for th
     * ``vistir.compat.JSONDecodeError``
     * ``vistir.compat.ResourceWarning``
     * ``vistir.compat.FileNotFoundError``
+    * ``vistir.compat.PermissionError``
+    * ``vistir.compat.IsADirectoryError``
 
-The following additional function is provided for encoding strings to the filesystem
-defualt encoding:
+The following additional functions are provided for encoding strings to the filesystem
+default encoding:
 
     * ``vistir.compat.fs_str``
+    * ``vistir.compat.to_native_string``
 
 
 🐉 Context Managers

docs/quickstart.rst

@@ -29,7 +29,7 @@ Install from `PyPI`_:
 
   ::
 
-    $ pipenv install --pre vistir
+    $ pipenv install vistir
 
 Install from `Github`_:
 
@@ -113,11 +113,14 @@ Shims are provided for full API compatibility from python 2.7 through 3.7 for th
     * :class:`~vistir.compat.JSONDecodeError`
     * :exc:`~vistir.compat.ResourceWarning`
     * :exc:`~vistir.compat.FileNotFoundError`
+    * :exc:`~vistir.compat.PermissionError`
+    * :exc:`~vistir.compat.IsADirectoryError`
 
 The following additional function is provided for encoding strings to the filesystem
 defualt encoding:
 
     * :func:`~vistir.compat.fs_str`
+    * :func:`~vistir.compat.to_native_string`
 
 
 🐉 Context Managers

news/38.feature.rst

@@ -0,0 +1 @@
+Backported compatibility shims from ``CPython`` for improved cleanup of readonly temporary directories on cleanup.

src/vistir/compat.py

@@ -19,7 +19,8 @@
     "JSONDecodeError",
     "FileNotFoundError",
     "ResourceWarning",
-    "FileNotFoundError",
+    "PermissionError",
+    "IsADirectoryError",
     "fs_str",
     "lru_cache",
     "TemporaryDirectory",
@@ -69,8 +70,17 @@ def __init__(self, *args, **kwargs):
             self.errno = errno.ENOENT
             super(FileNotFoundError, self).__init__(*args, **kwargs)
 
+    class PermissionError(OSError):
+        def __init__(self, *args, **kwargs):
+            self.errno = errno.EACCES
+            super(PermissionError, self).__init__(*args, **kwargs)
+
+    class IsADirectoryError(OSError):
+        """The command does not work on directories"""
+        pass
+
 else:
-    from builtins import ResourceWarning, FileNotFoundError
+    from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError
 
 
 if not sys.warnoptions:
@@ -111,9 +121,39 @@ def __init__(self, suffix="", prefix=None, dir=None):
         )
 
     @classmethod
-    def _cleanup(cls, name, warn_message):
+    def _rmtree(cls, name):
         from .path import rmtree
-        rmtree(name)
+
+        def onerror(func, path, exc_info):
+            if issubclass(exc_info[0], (PermissionError, OSError)):
+                try:
+                    try:
+                        if path != name:
+                            os.chflags(os.path.dirname(path), 0)
+                        os.chflags(path, 0)
+                    except AttributeError:
+                        pass
+                    if path != name:
+                        os.chmod(os.path.dirname(path), 0o70)
+                    os.chmod(path, 0o700)
+
+                    try:
+                        os.unlink(path)
+                    # PermissionError is raised on FreeBSD for directories
+                    except (IsADirectoryError, PermissionError, OSError):
+                        cls._rmtree(path)
+                except FileNotFoundError:
+                    pass
+            elif issubclass(exc_info[0], FileNotFoundError):
+                pass
+            else:
+                raise
+
+        rmtree(name, onerror=onerror)
+
+    @classmethod
+    def _cleanup(cls, name, warn_message):
+        cls._rmtree(name)
         warnings.warn(warn_message, ResourceWarning)
 
     def __repr__(self):
@@ -126,16 +166,16 @@ def __exit__(self, exc, value, tb):
         self.cleanup()
 
     def cleanup(self):
-        from .path import rmtree
         if self._finalizer.detach():
-            rmtree(self.name)
+            self._rmtree(self.name)
 
 
 def fs_str(string):
     """Encodes a string into the proper filesystem encoding
 
     Borrowed from pip-tools
     """
+
     if isinstance(string, str):
         return string
     assert not isinstance(string, bytes)

src/vistir/path.py

@@ -8,7 +8,6 @@
 import posixpath
 import shutil
 import stat
-import sys
 import warnings
 
 import six
@@ -276,22 +275,26 @@ def set_write_bit(fn):
     file_stat = os.stat(fn).st_mode
     os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
     if not os.path.isdir(fn):
-        return
+        try:
+            os.chflags(fn, 0)
+        except AttributeError:
+            pass
     for root, dirs, files in os.walk(fn, topdown=False):
         for dir_ in [os.path.join(root,d) for d in dirs]:
             set_write_bit(dir_)
         for file_ in [os.path.join(root, f) for f in files]:
             set_write_bit(file_)
 
 
-def rmtree(directory, ignore_errors=False):
+def rmtree(directory, ignore_errors=False, onerror=None):
     """Stand-in for :func:`~shutil.rmtree` with additional error-handling.
 
     This version of `rmtree` handles read-only paths, especially in the case of index
     files written by certain source control systems.
 
     :param str directory: The target directory to remove
     :param bool ignore_errors: Whether to ignore errors, defaults to False
+    :param func onerror: An error handling function, defaults to :func:`handle_remove_readonly`
 
     .. note::
 
@@ -301,9 +304,11 @@ def rmtree(directory, ignore_errors=False):
     from .compat import to_native_string
 
     directory = to_native_string(directory)
+    if onerror is None:
+        onerror = handle_remove_readonly
     try:
         shutil.rmtree(
-            directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly
+            directory, ignore_errors=ignore_errors, onerror=onerror
         )
     except (IOError, OSError, FileNotFoundError) as exc:
         # Ignore removal failures where the file doesn't exist
@@ -326,7 +331,9 @@ def handle_remove_readonly(func, path, exc):
     :func:`set_write_bit` on the target path and try again.
     """
     # Check for read-only attribute
-    from .compat import ResourceWarning, FileNotFoundError, to_native_string
+    from .compat import (
+        ResourceWarning, FileNotFoundError, PermissionError, to_native_string
+    )
 
     PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT)
     default_warning_message = (
@@ -340,7 +347,7 @@ def handle_remove_readonly(func, path, exc):
         set_write_bit(path)
         try:
             func(path)
-        except (OSError, IOError, FileNotFoundError) as e:
+        except (OSError, IOError, FileNotFoundError, PermissionError) as e:
             if e.errno == errno.ENOENT:
                 return
             elif e.errno in PERM_ERRORS:
@@ -351,7 +358,7 @@ def handle_remove_readonly(func, path, exc):
         set_write_bit(path)
         try:
             func(path)
-        except (OSError, IOError, FileNotFoundError) as e:
+        except (OSError, IOError, FileNotFoundError, PermissionError) as e:
             if e.errno in PERM_ERRORS:
                 warnings.warn(default_warning_message.format(path), ResourceWarning)
                 pass

src/vistir/spin.py

@@ -292,6 +292,6 @@ def _clear_line():
 def create_spinner(*args, **kwargs):
     nospin = kwargs.pop("nospin", False)
     use_yaspin = kwargs.pop("use_yaspin", nospin)
-    if nospin:
+    if nospin or not use_yaspin:
         return DummySpinner(*args, **kwargs)
     return VistirSpinner(*args, **kwargs)