Skip to content

Commit 6258844

Browse files
authored
GH-117586: Speed up pathlib.Path.glob() by working with strings (#117589)
Move pathlib globbing implementation into a new private class: `glob._Globber`. This class implements fast string-based globbing. It's called by `pathlib.Path.glob()`, which then converts strings back to path objects. In the private pathlib ABCs, add a `pathlib._abc.Globber` subclass that works with `PathBase` objects rather than strings, and calls user-defined path methods like `PathBase.stat()` rather than `os.stat()`. This sets the stage for two more improvements: - GH-115060: Query non-wildcard segments with `lstat()` - GH-116380: Unify `pathlib` and `glob` implementations of globbing. No change to the implementations of `glob.glob()` and `glob.iglob()`.
1 parent 689ada7 commit 6258844

File tree

4 files changed

+269
-195
lines changed

4 files changed

+269
-195
lines changed

Lib/glob.py

+186
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
import os
55
import re
66
import fnmatch
7+
import functools
78
import itertools
9+
import operator
810
import stat
911
import sys
1012

@@ -256,7 +258,9 @@ def escape(pathname):
256258
return drive + pathname
257259

258260

261+
_special_parts = ('', '.', '..')
259262
_dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)
263+
_no_recurse_symlinks = object()
260264

261265

262266
def translate(pat, *, recursive=False, include_hidden=False, seps=None):
@@ -312,3 +316,185 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None):
312316
results.append(any_sep)
313317
res = ''.join(results)
314318
return fr'(?s:{res})\Z'
319+
320+
321+
@functools.lru_cache(maxsize=512)
322+
def _compile_pattern(pat, sep, case_sensitive, recursive=True):
323+
"""Compile given glob pattern to a re.Pattern object (observing case
324+
sensitivity)."""
325+
flags = re.NOFLAG if case_sensitive else re.IGNORECASE
326+
regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep)
327+
return re.compile(regex, flags=flags).match
328+
329+
330+
class _Globber:
331+
"""Class providing shell-style pattern matching and globbing.
332+
"""
333+
334+
def __init__(self, sep, case_sensitive, recursive=False):
335+
self.sep = sep
336+
self.case_sensitive = case_sensitive
337+
self.recursive = recursive
338+
339+
# Low-level methods
340+
341+
lstat = staticmethod(os.lstat)
342+
scandir = staticmethod(os.scandir)
343+
parse_entry = operator.attrgetter('path')
344+
concat_path = operator.add
345+
346+
if os.name == 'nt':
347+
@staticmethod
348+
def add_slash(pathname):
349+
tail = os.path.splitroot(pathname)[2]
350+
if not tail or tail[-1] in '\\/':
351+
return pathname
352+
return f'{pathname}\\'
353+
else:
354+
@staticmethod
355+
def add_slash(pathname):
356+
if not pathname or pathname[-1] == '/':
357+
return pathname
358+
return f'{pathname}/'
359+
360+
# High-level methods
361+
362+
def compile(self, pat):
363+
return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive)
364+
365+
def selector(self, parts):
366+
"""Returns a function that selects from a given path, walking and
367+
filtering according to the glob-style pattern parts in *parts*.
368+
"""
369+
if not parts:
370+
return self.select_exists
371+
part = parts.pop()
372+
if self.recursive and part == '**':
373+
selector = self.recursive_selector
374+
elif part in _special_parts:
375+
selector = self.special_selector
376+
else:
377+
selector = self.wildcard_selector
378+
return selector(part, parts)
379+
380+
def special_selector(self, part, parts):
381+
"""Returns a function that selects special children of the given path.
382+
"""
383+
select_next = self.selector(parts)
384+
385+
def select_special(path, exists=False):
386+
path = self.concat_path(self.add_slash(path), part)
387+
return select_next(path, exists)
388+
return select_special
389+
390+
def wildcard_selector(self, part, parts):
391+
"""Returns a function that selects direct children of a given path,
392+
filtering by pattern.
393+
"""
394+
395+
match = None if part == '*' else self.compile(part)
396+
dir_only = bool(parts)
397+
if dir_only:
398+
select_next = self.selector(parts)
399+
400+
def select_wildcard(path, exists=False):
401+
try:
402+
# We must close the scandir() object before proceeding to
403+
# avoid exhausting file descriptors when globbing deep trees.
404+
with self.scandir(path) as scandir_it:
405+
entries = list(scandir_it)
406+
except OSError:
407+
pass
408+
else:
409+
for entry in entries:
410+
if match is None or match(entry.name):
411+
if dir_only:
412+
try:
413+
if not entry.is_dir():
414+
continue
415+
except OSError:
416+
continue
417+
entry_path = self.parse_entry(entry)
418+
if dir_only:
419+
yield from select_next(entry_path, exists=True)
420+
else:
421+
yield entry_path
422+
return select_wildcard
423+
424+
def recursive_selector(self, part, parts):
425+
"""Returns a function that selects a given path and all its children,
426+
recursively, filtering by pattern.
427+
"""
428+
# Optimization: consume following '**' parts, which have no effect.
429+
while parts and parts[-1] == '**':
430+
parts.pop()
431+
432+
# Optimization: consume and join any following non-special parts here,
433+
# rather than leaving them for the next selector. They're used to
434+
# build a regular expression, which we use to filter the results of
435+
# the recursive walk. As a result, non-special pattern segments
436+
# following a '**' wildcard don't require additional filesystem access
437+
# to expand.
438+
follow_symlinks = self.recursive is not _no_recurse_symlinks
439+
if follow_symlinks:
440+
while parts and parts[-1] not in _special_parts:
441+
part += self.sep + parts.pop()
442+
443+
match = None if part == '**' else self.compile(part)
444+
dir_only = bool(parts)
445+
select_next = self.selector(parts)
446+
447+
def select_recursive(path, exists=False):
448+
path = self.add_slash(path)
449+
match_pos = len(str(path))
450+
if match is None or match(str(path), match_pos):
451+
yield from select_next(path, exists)
452+
stack = [path]
453+
while stack:
454+
yield from select_recursive_step(stack, match_pos)
455+
456+
def select_recursive_step(stack, match_pos):
457+
path = stack.pop()
458+
try:
459+
# We must close the scandir() object before proceeding to
460+
# avoid exhausting file descriptors when globbing deep trees.
461+
with self.scandir(path) as scandir_it:
462+
entries = list(scandir_it)
463+
except OSError:
464+
pass
465+
else:
466+
for entry in entries:
467+
is_dir = False
468+
try:
469+
if entry.is_dir(follow_symlinks=follow_symlinks):
470+
is_dir = True
471+
except OSError:
472+
pass
473+
474+
if is_dir or not dir_only:
475+
entry_path = self.parse_entry(entry)
476+
if match is None or match(str(entry_path), match_pos):
477+
if dir_only:
478+
yield from select_next(entry_path, exists=True)
479+
else:
480+
# Optimization: directly yield the path if this is
481+
# last pattern part.
482+
yield entry_path
483+
if is_dir:
484+
stack.append(entry_path)
485+
486+
return select_recursive
487+
488+
def select_exists(self, path, exists=False):
489+
"""Yields the given path, if it exists.
490+
"""
491+
if exists:
492+
# Optimization: this path is already known to exist, e.g. because
493+
# it was returned from os.scandir(), so we skip calling lstat().
494+
yield path
495+
else:
496+
try:
497+
self.lstat(path)
498+
yield path
499+
except OSError:
500+
pass

Lib/pathlib/__init__.py

+47-30
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@
55
operating systems.
66
"""
77

8+
import glob
89
import io
910
import ntpath
11+
import operator
1012
import os
1113
import posixpath
1214
import sys
@@ -111,6 +113,7 @@ class PurePath(_abc.PurePathBase):
111113
'_hash',
112114
)
113115
parser = os.path
116+
_globber = glob._Globber
114117

115118
def __new__(cls, *args, **kwargs):
116119
"""Construct a PurePath from one or several strings and or existing
@@ -253,14 +256,17 @@ def _format_parsed_parts(cls, drv, root, tail):
253256
return cls.parser.sep.join(tail)
254257

255258
def _from_parsed_parts(self, drv, root, tail):
256-
path_str = self._format_parsed_parts(drv, root, tail)
257-
path = self.with_segments(path_str)
258-
path._str = path_str or '.'
259+
path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail))
259260
path._drv = drv
260261
path._root = root
261262
path._tail_cached = tail
262263
return path
263264

265+
def _from_parsed_string(self, path_str):
266+
path = self.with_segments(path_str)
267+
path._str = path_str or '.'
268+
return path
269+
264270
@classmethod
265271
def _parse_path(cls, path):
266272
if not path:
@@ -453,21 +459,6 @@ def as_uri(self):
453459
from urllib.parse import quote_from_bytes
454460
return prefix + quote_from_bytes(os.fsencode(path))
455461

456-
@property
457-
def _pattern_stack(self):
458-
"""Stack of path components, to be used with patterns in glob()."""
459-
parts = self._tail.copy()
460-
pattern = self._raw_path
461-
if self.anchor:
462-
raise NotImplementedError("Non-relative patterns are unsupported")
463-
elif not parts:
464-
raise ValueError("Unacceptable pattern: {!r}".format(pattern))
465-
elif pattern[-1] in (self.parser.sep, self.parser.altsep):
466-
# GH-65238: pathlib doesn't preserve trailing slash. Add it back.
467-
parts.append('')
468-
parts.reverse()
469-
return parts
470-
471462
@property
472463
def _pattern_str(self):
473464
"""The path expressed as a string, for use in pattern-matching."""
@@ -576,6 +567,17 @@ def write_text(self, data, encoding=None, errors=None, newline=None):
576567
encoding = io.text_encoding(encoding)
577568
return _abc.PathBase.write_text(self, data, encoding, errors, newline)
578569

570+
_remove_leading_dot = operator.itemgetter(slice(2, None))
571+
_remove_trailing_slash = operator.itemgetter(slice(-1))
572+
573+
def _filter_trailing_slash(self, paths):
574+
sep = self.parser.sep
575+
anchor_len = len(self.anchor)
576+
for path_str in paths:
577+
if len(path_str) > anchor_len and path_str[-1] == sep:
578+
path_str = path_str[:-1]
579+
yield path_str
580+
579581
def iterdir(self):
580582
"""Yield path objects of the directory contents.
581583
@@ -587,13 +589,9 @@ def iterdir(self):
587589
def _scandir(self):
588590
return os.scandir(self)
589591

590-
def _direntry_str(self, entry):
591-
# Transform an entry yielded from _scandir() into a path string.
592-
return entry.name if str(self) == '.' else entry.path
593-
594592
def _make_child_direntry(self, entry):
595593
# Transform an entry yielded from _scandir() into a path object.
596-
path_str = self._direntry_str(entry)
594+
path_str = entry.name if str(self) == '.' else entry.path
597595
path = self.with_segments(path_str)
598596
path._str = path_str
599597
path._drv = self.drive
@@ -626,8 +624,30 @@ def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False):
626624
sys.audit("pathlib.Path.glob", self, pattern)
627625
if not isinstance(pattern, PurePath):
628626
pattern = self.with_segments(pattern)
629-
return _abc.PathBase.glob(
630-
self, pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks)
627+
if pattern.anchor:
628+
raise NotImplementedError("Non-relative patterns are unsupported")
629+
parts = pattern._tail.copy()
630+
if not parts:
631+
raise ValueError("Unacceptable pattern: {!r}".format(pattern))
632+
raw = pattern._raw_path
633+
if raw[-1] in (self.parser.sep, self.parser.altsep):
634+
# GH-65238: pathlib doesn't preserve trailing slash. Add it back.
635+
parts.append('')
636+
if not self.is_dir():
637+
return iter([])
638+
select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks)
639+
root = str(self)
640+
paths = select(root, exists=True)
641+
642+
# Normalize results
643+
if root == '.':
644+
paths = map(self._remove_leading_dot, paths)
645+
if parts[-1] == '':
646+
paths = map(self._remove_trailing_slash, paths)
647+
elif parts[-1] == '**':
648+
paths = self._filter_trailing_slash(paths)
649+
paths = map(self._from_parsed_string, paths)
650+
return paths
631651

632652
def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False):
633653
"""Recursively yield all existing files (of any kind, including
@@ -638,8 +658,7 @@ def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False):
638658
if not isinstance(pattern, PurePath):
639659
pattern = self.with_segments(pattern)
640660
pattern = '**' / pattern
641-
return _abc.PathBase.glob(
642-
self, pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks)
661+
return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks)
643662

644663
def walk(self, top_down=True, on_error=None, follow_symlinks=False):
645664
"""Walk the directory tree from this directory, similar to os.walk()."""
@@ -669,9 +688,7 @@ def absolute(self):
669688
# of joining, and we exploit the fact that getcwd() returns a
670689
# fully-normalized string by storing it in _str. This is used to
671690
# implement Path.cwd().
672-
result = self.with_segments(cwd)
673-
result._str = cwd
674-
return result
691+
return self._from_parsed_string(cwd)
675692
drive, root, rel = os.path.splitroot(cwd)
676693
if not rel:
677694
return self._from_parsed_parts(drive, root, self._tail)

0 commit comments

Comments
 (0)