Added filter_glob and exclude_glob to fs.walk.Walker.

These extend the class by an option to include/exclude resources by their entire path, not just its last component.
To do so, fs.wildcard had to undergo a rework to remove the dependency on the `re` module.
Unit tests were added for all new/changed code.

Author: atollk

CHANGELOG.md

@@ -10,11 +10,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- To `fs.walk.Walker`, added parameters `filter_glob` and `exclude_glob`.
+  Closes [#459](https://github.com/PyFilesystem/pyfilesystem2/issues/459).
 - Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. 
   Closes [#458](https://github.com/PyFilesystem/pyfilesystem2/issues/458).
 
 ### Fixed
 
+- Elaborated documentation of `filter_dirs` and `exclude_dirs` in `fs.walk.Walker`.
+  Closes [#371](https://github.com/PyFilesystem/pyfilesystem2/issues/371).
 - Fixed performance bugs in `fs.copy.copy_dir_if_newer`. Test cases were adapted to catch those bugs in the future.
 
 

fs/base.py

@@ -1604,8 +1604,8 @@ def check(self):
         if self.isclosed():
             raise errors.FilesystemClosed()
 
-    def match(self, patterns, name):
-        # type: (Optional[Iterable[Text]], Text) -> bool
+    def match(self, patterns, name, accept_prefix=False):
+        # type: (Optional[Iterable[Text]], Text, bool) -> bool
         """Check if a name matches any of a list of wildcards.
 
         If a filesystem is case *insensitive* (such as Windows) then
@@ -1618,6 +1618,9 @@ def match(self, patterns, name):
             patterns (list, optional): A list of patterns, e.g.
                 ``['*.py']``, or `None` to match everything.
             name (str): A file or directory name (not a path)
+            accept_prefix (bool): If ``True``, the name is
+                not required to match the wildcards themselves
+                but only need to be a prefix of a string that does.
 
         Returns:
             bool: `True` if ``name`` matches any of the patterns.
@@ -1644,7 +1647,9 @@ def match(self, patterns, name):
         case_sensitive = not typing.cast(
             bool, self.getmeta().get("case_insensitive", False)
         )
-        matcher = wildcard.get_matcher(patterns, case_sensitive)
+        matcher = wildcard.get_matcher(
+            patterns, case_sensitive, accept_prefix=accept_prefix
+        )
         return matcher(name)
 
     def tree(self, **kwargs):

fs/glob.py

@@ -10,7 +10,6 @@
 from .lrucache import LRUCache
 from ._repr import make_repr
 from .path import iteratepath
-from . import wildcard
 
 
 GlobMatch = namedtuple("GlobMatch", ["path", "info"])
@@ -27,6 +26,54 @@
 )  # type: LRUCache[Tuple[Text, bool], Tuple[int, bool, Pattern]]
 
 
+def _translate(pattern, case_sensitive=True):
+    # type: (Text, bool) -> Text
+    """Translate a wildcard pattern to a regular expression.
+
+    There is no way to quote meta-characters.
+    Arguments:
+        pattern (str): A wildcard pattern.
+        case_sensitive (bool): Set to `False` to use a case
+            insensitive regex (default `True`).
+
+    Returns:
+        str: A regex equivalent to the given pattern.
+
+    """
+    if not case_sensitive:
+        pattern = pattern.lower()
+    i, n = 0, len(pattern)
+    res = ""
+    while i < n:
+        c = pattern[i]
+        i = i + 1
+        if c == "*":
+            res = res + "[^/]*"
+        elif c == "?":
+            res = res + "."
+        elif c == "[":
+            j = i
+            if j < n and pattern[j] == "!":
+                j = j + 1
+            if j < n and pattern[j] == "]":
+                j = j + 1
+            while j < n and pattern[j] != "]":
+                j = j + 1
+            if j >= n:
+                res = res + "\\["
+            else:
+                stuff = pattern[i:j].replace("\\", "\\\\")
+                i = j + 1
+                if stuff[0] == "!":
+                    stuff = "^" + stuff[1:]
+                elif stuff[0] == "^":
+                    stuff = "\\" + stuff
+                res = "%s[%s]" % (res, stuff)
+        else:
+            res = res + re.escape(c)
+    return res
+
+
 def _translate_glob(pattern, case_sensitive=True):
     levels = 0
     recursive = False
@@ -37,7 +84,7 @@ def _translate_glob(pattern, case_sensitive=True):
             recursive = True
         else:
             re_patterns.append(
-                "/" + wildcard._translate(component, case_sensitive=case_sensitive)
+                "/" + _translate(component, case_sensitive=case_sensitive)
             )
         levels += 1
     re_glob = "(?ms)^" + "".join(re_patterns) + ("/$" if pattern.endswith("/") else "$")

fs/walk.py

@@ -62,6 +62,8 @@ def __init__(
         filter_dirs=None,  # type: Optional[List[Text]]
         exclude_dirs=None,  # type: Optional[List[Text]]
         max_depth=None,  # type: Optional[int]
+        filter_glob=None,  # type: Optional[List[Text]]
+        exclude_glob=None,  # type: Optional[List[Text]]
     ):
         # type: (...) -> None
         """Create a new `Walker` instance.
@@ -85,11 +87,22 @@ def __init__(
                 any of these patterns will be removed from the walk.
             filter_dirs (list, optional): A list of patterns that will be used
                 to match directories paths. The walk will only open directories
-                that match at least one of these patterns.
+                that match at least one of these patterns. Directories will
+                only be returned if the final component matches one of the
+                patterns.
             exclude_dirs (list, optional): A list of patterns that will be
                 used to filter out directories from the walk. e.g.
-                ``['*.svn', '*.git']``.
+                ``['*.svn', '*.git']``. Directories matching any of these
+                patterns will be removed from the walk.
             max_depth (int, optional): Maximum directory depth to walk.
+            filter_glob (list, optional): If supplied, this parameter
+                should be a list of path patterns e.g. ``["foo/**/*.py"]``.
+                Resources will only be returned if their global path or
+                an extension of it matches one of the patterns.
+            exclude_glob (list, optional): If supplied, this parameter
+                should be a list of path patterns e.g. ``["foo/**/*.py"]``.
+                Resources will not be returned if their global path or
+                an extension of it  matches one of the patterns.
 
         """
         if search not in ("breadth", "depth"):
@@ -109,6 +122,8 @@ def __init__(
         self.exclude = exclude
         self.filter_dirs = filter_dirs
         self.exclude_dirs = exclude_dirs
+        self.filter_glob = filter_glob
+        self.exclude_glob = exclude_glob
         self.max_depth = max_depth
         super(Walker, self).__init__()
 
@@ -180,6 +195,8 @@ def __repr__(self):
             filter_dirs=(self.filter_dirs, None),
             exclude_dirs=(self.exclude_dirs, None),
             max_depth=(self.max_depth, None),
+            filter_glob=(self.filter_glob, None),
+            exclude_glob=(self.exclude_glob, None),
         )
 
     def _iter_walk(
@@ -198,9 +215,18 @@ def _iter_walk(
     def _check_open_dir(self, fs, path, info):
         # type: (FS, Text, Info) -> bool
         """Check if a directory should be considered in the walk."""
+        full_path = ("" if path == "/" else path) + "/" + info.name
         if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name):
             return False
-        if self.filter_dirs is not None and not fs.match(self.filter_dirs, info.name):
+        if self.exclude_glob is not None and fs.match(self.exclude_glob, full_path):
+            return False
+        if self.filter_dirs is not None and not fs.match(
+            self.filter_dirs, info.name, accept_prefix=True
+        ):
+            return False
+        if self.filter_glob is not None and not fs.match(
+            self.filter_glob, full_path, accept_prefix=True
+        ):
             return False
         return self.check_open_dir(fs, path, info)
 
@@ -247,6 +273,26 @@ def check_scan_dir(self, fs, path, info):
         """
         return True
 
+    def _check_file(self, fs, dir_path, info):
+        # type: (FS, Text, Info) -> bool
+        """Check if a filename should be included."""
+        # Weird check required for backwards compatibility,
+        # when _check_file did not exist.
+        if Walker._check_file == type(self)._check_file:
+            if self.exclude is not None and fs.match(self.exclude, info.name):
+                return False
+            if self.exclude_glob is not None and fs.match(
+                self.exclude_glob, dir_path + "/" + info.name
+            ):
+                return False
+            if self.filter is not None and not fs.match(self.filter, info.name):
+                return False
+            if self.filter_glob is not None and not fs.match(
+                self.filter_glob, dir_path + "/" + info.name, accept_prefix=True
+            ):
+                return False
+        return self.check_file(fs, info)
+
     def check_file(self, fs, info):
         # type: (FS, Info) -> bool
         """Check if a filename should be included.
@@ -261,9 +307,7 @@ def check_file(self, fs, info):
             bool: `True` if the file should be included.
 
         """
-        if self.exclude is not None and fs.match(self.exclude, info.name):
-            return False
-        return fs.match(self.filter, info.name)
+        return True
 
     def _scan(
         self,
@@ -420,7 +464,7 @@ def _walk_breadth(
         _calculate_depth = self._calculate_depth
         _check_open_dir = self._check_open_dir
         _check_scan_dir = self._check_scan_dir
-        _check_file = self.check_file
+        _check_file = self._check_file
 
         depth = _calculate_depth(path)
 
@@ -434,7 +478,7 @@ def _walk_breadth(
                         if _check_scan_dir(fs, dir_path, info, _depth):
                             push(_combine(dir_path, info.name))
                 else:
-                    if _check_file(fs, info):
+                    if _check_file(fs, dir_path, info):
                         yield dir_path, info  # Found a file
             yield dir_path, None  # End of directory
 
@@ -453,7 +497,7 @@ def _walk_depth(
         _calculate_depth = self._calculate_depth
         _check_open_dir = self._check_open_dir
         _check_scan_dir = self._check_scan_dir
-        _check_file = self.check_file
+        _check_file = self._check_file
         depth = _calculate_depth(path)
 
         stack = [
@@ -485,7 +529,7 @@ def _walk_depth(
                     else:
                         yield dir_path, info
             else:
-                if _check_file(fs, info):
+                if _check_file(fs, dir_path, info):
                     yield dir_path, info