glob.py

"""Filename globbing utility."""

importcontextlib
importos
importre
importfnmatch
importfunctools
importitertools
importoperator
importstat
importsys


__all__= ["glob", "iglob", "escape", "translate"]

defglob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
include_hidden=False):
"""Return a list of paths matching a pathname pattern.

 The pattern may contain simple shell-style wildcards a la
 fnmatch. Unlike fnmatch, filenames starting with a
 dot are special cases that are not matched by '*' and '?'
 patterns by default.

 If `include_hidden` is true, the patterns '*', '?', '**' will match hidden
 directories.

 If `recursive` is true, the pattern '**' will match any files and
 zero or more directories and subdirectories.
 """
returnlist(iglob(pathname, root_dir=root_dir, dir_fd=dir_fd, recursive=recursive,
include_hidden=include_hidden))

defiglob(pathname, *, root_dir=None, dir_fd=None, recursive=False,
include_hidden=False):
"""Return an iterator which yields the paths matching a pathname pattern.

 The pattern may contain simple shell-style wildcards a la
 fnmatch. However, unlike fnmatch, filenames starting with a
 dot are special cases that are not matched by '*' and '?'
 patterns.

 If recursive is true, the pattern '**' will match any files and
 zero or more directories and subdirectories.
 """
sys.audit("glob.glob", pathname, recursive)
sys.audit("glob.glob/2", pathname, recursive, root_dir, dir_fd)
ifroot_dirisnotNone:
root_dir=os.fspath(root_dir)
else:
root_dir=pathname[:0]
it=_iglob(pathname, root_dir, dir_fd, recursive, False,
include_hidden=include_hidden)
ifnotpathnameorrecursiveand_isrecursive(pathname[:2]):
try:
s=next(it) # skip empty string
ifs:
it=itertools.chain((s,), it)
exceptStopIteration:
pass
returnit

def_iglob(pathname, root_dir, dir_fd, recursive, dironly,
include_hidden=False):
dirname, basename=os.path.split(pathname)
ifnothas_magic(pathname):
assertnotdironly
ifbasename:
if_lexists(_join(root_dir, pathname), dir_fd):
yieldpathname
else:
# Patterns ending with a slash should match only directories
if_isdir(_join(root_dir, dirname), dir_fd):
yieldpathname
return
ifnotdirname:
ifrecursiveand_isrecursive(basename):
yieldfrom_glob2(root_dir, basename, dir_fd, dironly,
include_hidden=include_hidden)
else:
yieldfrom_glob1(root_dir, basename, dir_fd, dironly,
include_hidden=include_hidden)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
# contains magic characters (i.e. r'\\?\C:').
ifdirname!=pathnameandhas_magic(dirname):
dirs=_iglob(dirname, root_dir, dir_fd, recursive, True,
include_hidden=include_hidden)
else:
dirs= [dirname]
ifhas_magic(basename):
ifrecursiveand_isrecursive(basename):
glob_in_dir=_glob2
else:
glob_in_dir=_glob1
else:
glob_in_dir=_glob0
fordirnameindirs:
fornameinglob_in_dir(_join(root_dir, dirname), basename, dir_fd, dironly,
include_hidden=include_hidden):
yieldos.path.join(dirname, name)

# These 2 helper functions non-recursively glob inside a literal directory.
# They return a list of basenames. _glob1 accepts a pattern while _glob0
# takes a literal basename (so it only has to check for its existence).

def_glob1(dirname, pattern, dir_fd, dironly, include_hidden=False):
names=_listdir(dirname, dir_fd, dironly)
ifnot (include_hiddenor_ishidden(pattern)):
names= (xforxinnamesifnot_ishidden(x))
returnfnmatch.filter(names, pattern)

def_glob0(dirname, basename, dir_fd, dironly, include_hidden=False):
ifbasename:
if_lexists(_join(dirname, basename), dir_fd):
return [basename]
else:
# `os.path.split()` returns an empty basename for paths ending with a
# directory separator. 'q*x/' should match only directories.
if_isdir(dirname, dir_fd):
return [basename]
return []

_deprecated_function_message= (
"{name} is deprecated and will be removed in Python {remove}. Use "
"glob.glob and pass a directory to its root_dir argument instead."
)

defglob0(dirname, pattern):
importwarnings
warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15))
return_glob0(dirname, pattern, None, False)

defglob1(dirname, pattern):
importwarnings
warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15))
return_glob1(dirname, pattern, None, False)

# This helper function recursively yields relative pathnames inside a literal
# directory.

def_glob2(dirname, pattern, dir_fd, dironly, include_hidden=False):
assert_isrecursive(pattern)
ifnotdirnameor_isdir(dirname, dir_fd):
yieldpattern[:0]
yieldfrom_rlistdir(dirname, dir_fd, dironly,
include_hidden=include_hidden)

# If dironly is false, yields all file names inside a directory.
# If dironly is true, yields only directory names.
def_iterdir(dirname, dir_fd, dironly):
try:
fd=None
fsencode=None
ifdir_fdisnotNone:
ifdirname:
fd=arg=os.open(dirname, _dir_open_flags, dir_fd=dir_fd)
else:
arg=dir_fd
ifisinstance(dirname, bytes):
fsencode=os.fsencode
elifdirname:
arg=dirname
elifisinstance(dirname, bytes):
arg=bytes(os.curdir, 'ASCII')
else:
arg=os.curdir
try:
withos.scandir(arg) asit:
forentryinit:
try:
ifnotdironlyorentry.is_dir():
iffsencodeisnotNone:
yieldfsencode(entry.name)
else:
yieldentry.name
exceptOSError:
pass
finally:
iffdisnotNone:
os.close(fd)
exceptOSError:
return

def_listdir(dirname, dir_fd, dironly):
withcontextlib.closing(_iterdir(dirname, dir_fd, dironly)) asit:
returnlist(it)

# Recursively yields relative pathnames inside a literal directory.
def_rlistdir(dirname, dir_fd, dironly, include_hidden=False):
names=_listdir(dirname, dir_fd, dironly)
forxinnames:
ifinclude_hiddenornot_ishidden(x):
yieldx
path=_join(dirname, x) ifdirnameelsex
foryin_rlistdir(path, dir_fd, dironly,
include_hidden=include_hidden):
yield_join(x, y)


def_lexists(pathname, dir_fd):
# Same as os.path.lexists(), but with dir_fd
ifdir_fdisNone:
returnos.path.lexists(pathname)
try:
os.lstat(pathname, dir_fd=dir_fd)
except (OSError, ValueError):
returnFalse
else:
returnTrue

def_isdir(pathname, dir_fd):
# Same as os.path.isdir(), but with dir_fd
ifdir_fdisNone:
returnos.path.isdir(pathname)
try:
st=os.stat(pathname, dir_fd=dir_fd)
except (OSError, ValueError):
returnFalse
else:
returnstat.S_ISDIR(st.st_mode)

def_join(dirname, basename):
# It is common if dirname or basename is empty
ifnotdirnameornotbasename:
returndirnameorbasename
returnos.path.join(dirname, basename)

magic_check=re.compile('([*?[])')
magic_check_bytes=re.compile(b'([*?[])')

defhas_magic(s):
ifisinstance(s, bytes):
match=magic_check_bytes.search(s)
else:
match=magic_check.search(s)
returnmatchisnotNone

def_ishidden(path):
returnpath[0] in ('.', b'.'[0])

def_isrecursive(pattern):
ifisinstance(pattern, bytes):
returnpattern==b'**'
else:
returnpattern=='**'

defescape(pathname):
"""Escape all special characters.
 """
# Escaping is done by wrapping any of "*?[" between square brackets.
# Metacharacters do not work in the drive part and shouldn't be escaped.
drive, pathname=os.path.splitdrive(pathname)
ifisinstance(pathname, bytes):
pathname=magic_check_bytes.sub(br'[\1]', pathname)
else:
pathname=magic_check.sub(r'[\1]', pathname)
returndrive+pathname


_special_parts= ('', '.', '..')
_dir_open_flags=os.O_RDONLY|getattr(os, 'O_DIRECTORY', 0)
_no_recurse_symlinks=object()


deftranslate(pat, *, recursive=False, include_hidden=False, seps=None):
"""Translate a pathname with shell wildcards to a regular expression.

 If `recursive` is true, the pattern segment '**' will match any number of
 path segments.

 If `include_hidden` is true, wildcards can match path segments beginning
 with a dot ('.').

 If a sequence of separator characters is given to `seps`, they will be
 used to split the pattern into segments and match path separators. If not
 given, os.path.sep and os.path.altsep (where available) are used.
 """
ifnotseps:
ifos.path.altsep:
seps= (os.path.sep, os.path.altsep)
else:
seps=os.path.sep
escaped_seps=''.join(map(re.escape, seps))
any_sep=f'[{escaped_seps}]'iflen(seps) >1elseescaped_seps
not_sep=f'[^{escaped_seps}]'
ifinclude_hidden:
one_last_segment=f'{not_sep}+'
one_segment=f'{one_last_segment}{any_sep}'
any_segments=f'(?:.+{any_sep})?'
any_last_segments='.*'
else:
one_last_segment=f'[^{escaped_seps}.]{not_sep}*'
one_segment=f'{one_last_segment}{any_sep}'
any_segments=f'(?:{one_segment})*'
any_last_segments=f'{any_segments}(?:{one_last_segment})?'

results= []
parts=re.split(any_sep, pat)
last_part_idx=len(parts) -1
foridx, partinenumerate(parts):
ifpart=='*':
results.append(one_segmentifidx<last_part_idxelseone_last_segment)
elifrecursiveandpart=='**':
ifidx<last_part_idx:
ifparts[idx+1] !='**':
results.append(any_segments)
else:
results.append(any_last_segments)
else:
ifpart:
ifnotinclude_hiddenandpart[0] in'*?':
results.append(r'(?!\.)')
results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep))
ifidx<last_part_idx:
results.append(any_sep)
res=''.join(results)
returnfr'(?s:{res})\Z'


@functools.lru_cache(maxsize=512)
def_compile_pattern(pat, sep, case_sensitive, recursive=True):
"""Compile given glob pattern to a re.Pattern object (observing case
 sensitivity)."""
flags=re.NOFLAGifcase_sensitiveelsere.IGNORECASE
regex=translate(pat, recursive=recursive, include_hidden=True, seps=sep)
returnre.compile(regex, flags=flags).match


class_Globber:
"""Class providing shell-style pattern matching and globbing.
 """

def__init__(self, sep, case_sensitive, case_pedantic=False, recursive=False):
self.sep=sep
self.case_sensitive=case_sensitive
self.case_pedantic=case_pedantic
self.recursive=recursive

# Low-level methods

lstat=operator.methodcaller('lstat')
add_slash=operator.methodcaller('joinpath', '')

@staticmethod
defscandir(path):
"""Emulates os.scandir(), which returns an object that can be used as
 a context manager. This method is called by walk() and glob().
 """
returncontextlib.nullcontext(path.iterdir())

@staticmethod
defconcat_path(path, text):
"""Appends text to the given path.
 """
returnpath.with_segments(path._raw_path+text)

@staticmethod
defparse_entry(entry):
"""Returns the path of an entry yielded from scandir().
 """
returnentry

# High-level methods

defcompile(self, pat):
return_compile_pattern(pat, self.sep, self.case_sensitive, self.recursive)

defselector(self, parts):
"""Returns a function that selects from a given path, walking and
 filtering according to the glob-style pattern parts in *parts*.
 """
ifnotparts:
returnself.select_exists
part=parts.pop()
ifself.recursiveandpart=='**':
selector=self.recursive_selector
elifpartin_special_parts:
selector=self.special_selector
elifnotself.case_pedanticandmagic_check.search(part) isNone:
selector=self.literal_selector
else:
selector=self.wildcard_selector
returnselector(part, parts)

defspecial_selector(self, part, parts):
"""Returns a function that selects special children of the given path.
 """
select_next=self.selector(parts)

defselect_special(path, exists=False):
path=self.concat_path(self.add_slash(path), part)
returnselect_next(path, exists)
returnselect_special

defliteral_selector(self, part, parts):
"""Returns a function that selects a literal descendant of a path.
 """

# Optimization: consume and join any subsequent literal parts here,
# rather than leaving them for the next selector. This reduces the
# number of string concatenation operations and calls to add_slash().
whilepartsandmagic_check.search(parts[-1]) isNone:
part+=self.sep+parts.pop()

select_next=self.selector(parts)

defselect_literal(path, exists=False):
path=self.concat_path(self.add_slash(path), part)
returnselect_next(path, exists=False)
returnselect_literal

defwildcard_selector(self, part, parts):
"""Returns a function that selects direct children of a given path,
 filtering by pattern.
 """

match=Noneifpart=='*'elseself.compile(part)
dir_only=bool(parts)
ifdir_only:
select_next=self.selector(parts)

defselect_wildcard(path, exists=False):
try:
# We must close the scandir() object before proceeding to
# avoid exhausting file descriptors when globbing deep trees.
withself.scandir(path) asscandir_it:
entries=list(scandir_it)
exceptOSError:
pass
else:
forentryinentries:
ifmatchisNoneormatch(entry.name):
ifdir_only:
try:
ifnotentry.is_dir():
continue
exceptOSError:
continue
entry_path=self.parse_entry(entry)
ifdir_only:
yieldfromselect_next(entry_path, exists=True)
else:
yieldentry_path
returnselect_wildcard

defrecursive_selector(self, part, parts):
"""Returns a function that selects a given path and all its children,
 recursively, filtering by pattern.
 """
# Optimization: consume following '**' parts, which have no effect.
whilepartsandparts[-1] =='**':
parts.pop()

# Optimization: consume and join any following non-special parts here,
# rather than leaving them for the next selector. They're used to
# build a regular expression, which we use to filter the results of
# the recursive walk. As a result, non-special pattern segments
# following a '**' wildcard don't require additional filesystem access
# to expand.
follow_symlinks=self.recursiveisnot_no_recurse_symlinks
iffollow_symlinks:
whilepartsandparts[-1] notin_special_parts:
part+=self.sep+parts.pop()

match=Noneifpart=='**'elseself.compile(part)
dir_only=bool(parts)
select_next=self.selector(parts)

defselect_recursive(path, exists=False):
path=self.add_slash(path)
match_pos=len(str(path))
ifmatchisNoneormatch(str(path), match_pos):
yieldfromselect_next(path, exists)
stack= [path]
whilestack:
yieldfromselect_recursive_step(stack, match_pos)

defselect_recursive_step(stack, match_pos):
path=stack.pop()
try:
# We must close the scandir() object before proceeding to
# avoid exhausting file descriptors when globbing deep trees.
withself.scandir(path) asscandir_it:
entries=list(scandir_it)
exceptOSError:
pass
else:
forentryinentries:
is_dir=False
try:
ifentry.is_dir(follow_symlinks=follow_symlinks):
is_dir=True
exceptOSError:
pass

ifis_dirornotdir_only:
entry_path=self.parse_entry(entry)
ifmatchisNoneormatch(str(entry_path), match_pos):
ifdir_only:
yieldfromselect_next(entry_path, exists=True)
else:
# Optimization: directly yield the path if this is
# last pattern part.
yieldentry_path
ifis_dir:
stack.append(entry_path)

returnselect_recursive

defselect_exists(self, path, exists=False):
"""Yields the given path, if it exists.
 """
ifexists:
# Optimization: this path is already known to exist, e.g. because
# it was returned from os.scandir(), so we skip calling lstat().
yieldpath
else:
try:
self.lstat(path)
yieldpath
exceptOSError:
pass


class_StringGlobber(_Globber):
lstat=staticmethod(os.lstat)
scandir=staticmethod(os.scandir)
parse_entry=operator.attrgetter('path')
concat_path=operator.add

ifos.name=='nt':
@staticmethod
defadd_slash(pathname):
tail=os.path.splitroot(pathname)[2]
ifnottailortail[-1] in'\\/':
returnpathname
returnf'{pathname}\\'
else:
@staticmethod
defadd_slash(pathname):
ifnotpathnameorpathname[-1] =='/':
returnpathname
returnf'{pathname}/'