Added filter_glob and exclude_glob to fs.walk.Walker.

These extend the class by an option to include/exclude resources by their entire path, not just its last component.
To do so, fs.wildcard had to undergo a rework to remove the dependency on the `re` module.
Unit tests were added for all new/changed code.

Author: atollk

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- To `fs.walk.Walker`, added parameters `filter_glob` and `exclude_glob`.
+  Closes [#459](https://github.com/PyFilesystem/pyfilesystem2/issues/459).
 - Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. 
   Closes [#458](https://github.com/PyFilesystem/pyfilesystem2/issues/458).
 - Added `fs.base.FS.getmodified`.
@@ -22,6 +24,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+- Elaborated documentation of `filter_dirs` and `exclude_dirs` in `fs.walk.Walker`.
+  Closes [#371](https://github.com/PyFilesystem/pyfilesystem2/issues/371).
 - Fixed performance bugs in `fs.copy.copy_dir_if_newer`. Test cases were adapted to catch those bugs in the future.
 - Fixed precision bug for timestamps in `fs.OSFS.setinfo`.
 

fs/base.py

@@ -21,7 +21,7 @@
 
 import six
 
-from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard
+from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard, glob
 from .copy import copy_modified_time
 from .glob import BoundGlobber
 from .mode import validate_open_mode
@@ -1648,8 +1648,8 @@ def check(self):
         if self.isclosed():
             raise errors.FilesystemClosed()
 
-    def match(self, patterns, name):
-        # type: (Optional[Iterable[Text]], Text) -> bool
+    def match(self, patterns, name, accept_prefix=False):
+        # type: (Optional[Iterable[Text]], Text, bool) -> bool
         """Check if a name matches any of a list of wildcards.
 
         If a filesystem is case *insensitive* (such as Windows) then
@@ -1691,6 +1691,61 @@ def match(self, patterns, name):
         matcher = wildcard.get_matcher(patterns, case_sensitive)
         return matcher(name)
 
+    def match_glob(self, patterns, path, accept_prefix=False):
+        # type: (Optional[Iterable[Text]], Text, bool) -> bool
+        """Check if a path matches any of a list of glob patterns.
+
+        If a filesystem is case *insensitive* (such as Windows) then
+        this method will perform a case insensitive match (i.e. ``*.py``
+        will match the same names as ``*.PY``). Otherwise the match will
+        be case sensitive (``*.py`` and ``*.PY`` will match different
+        names).
+
+        Arguments:
+            patterns (list, optional): A list of patterns, e.g.
+                ``['*.py']``, or `None` to match everything.
+            path (str): A resource path, starting with "/".
+            accept_prefix (bool): If ``True``, the path is
+                not required to match the wildcards themselves
+                but only need to be a prefix of a string that does.
+
+        Returns:
+            bool: `True` if ``path`` matches any of the patterns.
+
+        Raises:
+            TypeError: If ``patterns`` is a single string instead of
+                a list (or `None`).
+            ValueError: If ``path`` is not a string starting with "/".
+
+        Example:
+            >>> my_fs.match_glob(['*.py'], '/__init__.py')
+            True
+            >>> my_fs.match_glob(['*.jpg', '*.png'], '/foo.gif')
+            False
+            >>> my_fs.match_glob(['dir/file.txt'], '/dir/', accept_prefix=True)
+            True
+            >>> my_fs.match_glob(['dir/file.txt'], '/dir/gile.txt', accept_prefix=True)
+            False
+
+        Note:
+            If ``patterns`` is `None` (or ``['*']``), then this
+            method will always return `True`.
+
+        """
+        if patterns is None:
+            return True
+        if not path or path[0] != "/":
+            raise ValueError("%s needs to be a string starting with /" % path)
+        if isinstance(patterns, six.text_type):
+            raise TypeError("patterns must be a list or sequence")
+        case_sensitive = not typing.cast(
+            bool, self.getmeta().get("case_insensitive", False)
+        )
+        matcher = glob.get_matcher(
+            patterns, case_sensitive, accept_prefix=accept_prefix
+        )
+        return matcher(path)
+
     def tree(self, **kwargs):
         # type: (**Any) -> None
         """Render a tree view of the filesystem to stdout or a file.

fs/errors.py

@@ -42,6 +42,7 @@
     "OperationFailed",
     "OperationTimeout",
     "PathError",
+    "PatternError",
     "PermissionDenied",
     "RemoteConnectionError",
     "RemoveRootError",
@@ -51,6 +52,7 @@
     "ResourceNotFound",
     "ResourceReadOnly",
     "Unsupported",
+    "UnsupportedHash",
 ]
 
 
@@ -346,3 +348,19 @@ class UnsupportedHash(ValueError):
     not supported by hashlib.
 
     """
+
+
+class PatternError(ValueError):
+    """A string pattern with invalid syntax was given."""
+
+    default_message = "pattern '{pattern}' is invalid at position {position}"
+
+    def __init__(self, pattern, position, exc=None, msg=None):  # noqa: D107
+        # type: (Text, int, Optional[Exception], Optional[Text]) -> None
+        self.pattern = pattern
+        self.position = position
+        self.exc = exc
+        super(ValueError, self).__init__()
+
+    def __reduce__(self):
+        return type(self), (self.path, self.position, self.exc, self._msg)

fs/glob.py

@@ -6,19 +6,28 @@
 from collections import namedtuple
 import re
 import typing
+from functools import partial
 
 from .lrucache import LRUCache
 from ._repr import make_repr
 from .path import iteratepath
-from . import wildcard
 
 
 GlobMatch = namedtuple("GlobMatch", ["path", "info"])
 Counts = namedtuple("Counts", ["files", "directories", "data"])
 LineCounts = namedtuple("LineCounts", ["lines", "non_blank"])
 
 if typing.TYPE_CHECKING:
-    from typing import Iterator, List, Optional, Pattern, Text, Tuple
+    from typing import (
+        Iterator,
+        List,
+        Optional,
+        Pattern,
+        Text,
+        Tuple,
+        Iterable,
+        Callable,
+    )
     from .base import FS
 
 
@@ -27,17 +36,87 @@
 )  # type: LRUCache[Tuple[Text, bool], Tuple[int, bool, Pattern]]
 
 
+def _split_pattern_by_rec(pattern):
+    # type: (Text) -> List[Text]
+    """Split a glob pattern at its directory seperators (/).
+
+    Takes into account escaped cases like [/].
+    """
+    indices = [-1]
+    bracket_open = False
+    for i, c in enumerate(pattern):
+        if c == "/" and not bracket_open:
+            indices.append(i)
+        elif c == "[":
+            bracket_open = True
+        elif c == "]":
+            bracket_open = False
+
+    indices.append(len(pattern))
+    return [pattern[i + 1 : j] for i, j in zip(indices[:-1], indices[1:])]
+
+
+def _translate(pattern, case_sensitive=True):
+    # type: (Text, bool) -> Text
+    """Translate a wildcard pattern to a regular expression.
+
+    There is no way to quote meta-characters.
+    Arguments:
+        pattern (str): A wildcard pattern.
+        case_sensitive (bool): Set to `False` to use a case
+            insensitive regex (default `True`).
+
+    Returns:
+        str: A regex equivalent to the given pattern.
+
+    """
+    if not case_sensitive:
+        pattern = pattern.lower()
+    i, n = 0, len(pattern)
+    res = []
+    while i < n:
+        c = pattern[i]
+        i = i + 1
+        if c == "*":
+            res.append("[^/]*")
+        elif c == "?":
+            res.append("[^/]")
+        elif c == "[":
+            j = i
+            if j < n and pattern[j] == "!":
+                j = j + 1
+            if j < n and pattern[j] == "]":
+                j = j + 1
+            while j < n and pattern[j] != "]":
+                j = j + 1
+            if j >= n:
+                res.append("\\[")
+            else:
+                stuff = pattern[i:j].replace("\\", "\\\\")
+                i = j + 1
+                if stuff[0] == "!":
+                    stuff = "^" + stuff[1:]
+                elif stuff[0] == "^":
+                    stuff = "\\" + stuff
+                res.append("[%s]" % stuff)
+        else:
+            res.append(re.escape(c))
+    return "".join(res)
+
+
 def _translate_glob(pattern, case_sensitive=True):
     levels = 0
     recursive = False
     re_patterns = [""]
     for component in iteratepath(pattern):
-        if component == "**":
-            re_patterns.append(".*/?")
+        if "**" in component:
             recursive = True
+            split = component.split("**")
+            split_re = [_translate(s, case_sensitive=case_sensitive) for s in split]
+            re_patterns.append("/?" + ".*/?".join(split_re))
         else:
             re_patterns.append(
-                "/" + wildcard._translate(component, case_sensitive=case_sensitive)
+                "/" + _translate(component, case_sensitive=case_sensitive)
             )
         levels += 1
     re_glob = "(?ms)^" + "".join(re_patterns) + ("/$" if pattern.endswith("/") else "$")
@@ -71,6 +150,8 @@ def match(pattern, path):
     except KeyError:
         levels, recursive, re_pattern = _translate_glob(pattern, case_sensitive=True)
         _PATTERN_CACHE[(pattern, True)] = (levels, recursive, re_pattern)
+    if path and path[0] != "/":
+        path = "/" + path
     return bool(re_pattern.match(path))
 
 
@@ -91,9 +172,95 @@ def imatch(pattern, path):
     except KeyError:
         levels, recursive, re_pattern = _translate_glob(pattern, case_sensitive=True)
         _PATTERN_CACHE[(pattern, False)] = (levels, recursive, re_pattern)
+    if path and path[0] != "/":
+        path = "/" + path
     return bool(re_pattern.match(path))
 
 
+def match_any(patterns, path):
+    # type: (Iterable[Text], Text) -> bool
+    """Test if a path matches any of a list of patterns.
+
+    Will return `True` if ``patterns`` is an empty list.
+
+    Arguments:
+        patterns (list): A list of wildcard pattern, e.g ``["*.py",
+            "*.pyc"]``
+        name (str): A filename.
+
+    Returns:
+        bool: `True` if the path matches at least one of the patterns.
+
+    """
+    if not patterns:
+        return True
+    return any(match(pattern, path) for pattern in patterns)
+
+
+def imatch_any(patterns, path):
+    # type: (Iterable[Text], Text) -> bool
+    """Test if a path matches any of a list of patterns (case insensitive).
+
+    Will return `True` if ``patterns`` is an empty list.
+
+    Arguments:
+        patterns (list): A list of wildcard pattern, e.g ``["*.py",
+            "*.pyc"]``
+        name (str): A filename.
+
+    Returns:
+        bool: `True` if the path matches at least one of the patterns.
+
+    """
+    if not patterns:
+        return True
+    return any(imatch(pattern, path) for pattern in patterns)
+
+
+def get_matcher(patterns, case_sensitive, accept_prefix=False):
+    # type: (Iterable[Text], bool, bool) -> Callable[[Text], bool]
+    """Get a callable that matches paths against the given patterns.
+
+    Arguments:
+        patterns (list): A list of wildcard pattern. e.g. ``["*.py",
+            "*.pyc"]``
+        case_sensitive (bool): If ``True``, then the callable will be case
+            sensitive, otherwise it will be case insensitive.
+        accept_prefix (bool): If ``True``, the name is
+            not required to match the wildcards themselves
+            but only need to be a prefix of a string that does.
+
+    Returns:
+        callable: a matcher that will return `True` if the paths given as
+        an argument matches any of the given patterns.
+
+    Example:
+        >>> from fs import wildcard
+        >>> is_python = wildcard.get_matcher(['*.py'], True)
+        >>> is_python('__init__.py')
+        True
+        >>> is_python('foo.txt')
+        False
+
+    """
+    if not patterns:
+        return lambda name: True
+
+    if accept_prefix:
+        new_patterns = []
+        for pattern in patterns:
+            split = _split_pattern_by_rec(pattern)
+            for i in range(1, len(split)):
+                new_pattern = "/".join(split[:i])
+                new_patterns.append(new_pattern)
+                new_patterns.append(new_pattern + "/")
+            new_patterns.append(pattern)
+        patterns = new_patterns
+
+    matcher = match_any if case_sensitive else imatch_any
+    return partial(matcher, patterns)
+
+
 class Globber(object):
     """A generator of glob results."""