Merge pull request #23: New timestamp pattern option

The pull request wasn't exactly complete (the code couldn't have run as
written, although it showed the general idea clear enough) so I decided
to treat #23 as more of a feature suggestion. However there was no
reason no to merge the pull request and use it as a base for my changes,
hence why I decided to do so despite rewriting the code.

Changes from the pull request:

- Renamed 'timestamp' to 'timestamp_pattern' (less ambiguous).

- Added validation that custom patterns define named capture
  groups corresponding to all of the required date components.

- Rewrote mapping from capture groups to datetime.datetime() arguments:

  - Previously positional datetime.datetime() arguments were used which
    depended on the order of capture groups in the hard coded regular
    expression pattern to function correctly.

  - Now that users can define their own patterns, this is no longer a
    reasonable approach. As such the code now constructs and passes a
    dictionary of keyword arguments to datetime.datetime().

- Updated documentation and command line interface usage message.

- Added tests for the new behavior.

Author: xolox

README.rst

@@ -153,6 +153,10 @@ intended you have no right to complain ;-).
    usage of the ``-H``, ``--hourly`` option for details about ``COUNT``."
    "``-y``, ``--yearly=COUNT``","Set the number of yearly backups to preserve during rotation. Refer to the
    usage of the ``-H``, ``--hourly`` option for details about ``COUNT``."
+   "``-t``, ``--timestamp-pattern=PATTERN``","Customize the regular expression pattern that is used to match and extract
+   timestamps from filenames. ``PATTERN`` is expected to be a Python compatible
+   regular expression that must define the named capture groups 'year',
+   'month' and 'day' and may define 'hour', 'minute' and 'second'."
    "``-I``, ``--include=PATTERN``","Only process backups that match the shell pattern given by ``PATTERN``. This
    argument can be repeated. Make sure to quote ``PATTERN`` so the shell doesn't
    expand the pattern before it's received by rotate-backups."
@@ -368,6 +372,28 @@ Supported configuration options
   ``weekly``, ``monthly`` and ``yearly`` options, these options support the
   same values as documented for the command line interface.
 
+- The ``timestamp-pattern`` option can be used to customize the regular
+  expression that's used to extract timestamps from filenames. The value is
+  expected to be a Python compatible regular expression that must contain the
+  named capture groups 'year', 'month' and 'day' and may contain the groups
+  'hour', 'minute' and 'second'. As an example here is the default regular
+  expression::
+
+    # Required components.
+    (?P<year>\d{4} ) \D?
+    (?P<month>\d{2}) \D?
+    (?P<day>\d{2}  ) \D?
+    (?:
+        # Optional components.
+        (?P<hour>\d{2}  ) \D?
+        (?P<minute>\d{2}) \D?
+        (?P<second>\d{2})?
+    )?
+
+  Note how this pattern spans multiple lines: Regular expressions are compiled
+  using the `re.VERBOSE`_ flag which means whitespace (including newlines) is
+  ignored.
+
 - The ``include-list`` and ``exclude-list`` options define a comma separated
   list of filename patterns to include or exclude, respectively:
 
@@ -384,7 +410,7 @@ Supported configuration options
   used to remove backups.
 
 - The ``ionice`` option expects one of the I/O scheduling class names ``idle``,
-  ``best-effort`` or ``realtime``.
+  ``best-effort`` or ``realtime`` (or the corresponding numbers).
 
 - The ``ssh-user`` option can be used to override the name of the remote SSH
   account that's used to connect to a remote system.
@@ -436,6 +462,7 @@ This software is licensed under the `MIT license`_.
 .. _peter@peterodding.com: peter@peterodding.com
 .. _PyPI: https://pypi.python.org/pypi/rotate-backups
 .. _Python Package Index: https://pypi.python.org/pypi/rotate-backups
+.. _re.VERBOSE: https://docs.python.org/3/library/re.html#re.VERBOSE
 .. _Read the Docs: https://rotate-backups.readthedocs.org
 .. _rsync: http://en.wikipedia.org/wiki/rsync
 .. _virtual environments: http://docs.python-guide.org/en/latest/dev/virtualenvs/

docs/conf.py

@@ -74,6 +74,7 @@
     'python3': ('https://docs.python.org/3/', None),
     'dateutil': ('https://dateutil.readthedocs.io/en/latest/', None),
     'executor': ('https://executor.readthedocs.io/en/latest/', None),
+    'humanfriendly': ('https://humanfriendly.readthedocs.io/en/latest/', None),
     'propertymanager': ('https://property-manager.readthedocs.io/en/latest/', None),
     'updatedotdee': ('https://update-dotdee.readthedocs.io/en/latest/', None),
 }

rotate_backups/__init__.py

@@ -1,7 +1,7 @@
 # rotate-backups: Simple command line interface for backup rotation.
 #
 # Author: Peter Odding <peter@peterodding.com>
-# Last Change: February 13, 2020
+# Last Change: February 14, 2020
 # URL: https://github.com/xolox/python-rotate-backups
 
 """
@@ -26,7 +26,7 @@
 from executor import ExternalCommandFailed
 from executor.concurrent import CommandPool
 from executor.contexts import RemoteContext, create_context
-from humanfriendly import Timer, coerce_boolean, format_path, parse_path, pluralize
+from humanfriendly import Timer, coerce_boolean, coerce_pattern, format_path, parse_path, pluralize
 from humanfriendly.text import concatenate, split
 from natsort import natsort
 from property_manager import (
@@ -36,6 +36,7 @@
     lazy_property,
     mutable_property,
     required_property,
+    set_property,
 )
 from simpleeval import simple_eval
 from six import string_types
@@ -48,6 +49,9 @@
 # Initialize a logger for this module.
 logger = VerboseLogger(__name__)
 
+DEFAULT_REMOVAL_COMMAND = ['rm', '-fR']
+"""The default removal command (a list of strings)."""
+
 ORDERED_FREQUENCIES = (
     ('minutely', relativedelta(minutes=1)),
     ('hourly', relativedelta(hours=1)),
@@ -57,14 +61,29 @@
     ('yearly', relativedelta(years=1)),
 )
 """
-A list of tuples with two values each:
+An iterable of tuples with two values each:
 
 - The name of a rotation frequency (a string like 'hourly', 'daily', etc.).
 - A :class:`~dateutil.relativedelta.relativedelta` object.
 
 The tuples are sorted by increasing delta (intentionally).
 """
 
+SUPPORTED_DATE_COMPONENTS = (
+    ('year', True),
+    ('month', True),
+    ('day', True),
+    ('hour', False),
+    ('minute', False),
+    ('second', False),
+)
+"""
+An iterable of tuples with two values each:
+
+- The name of a date component (a string).
+- :data:`True` for required components, :data:`False` for optional components.
+"""
+
 SUPPORTED_FREQUENCIES = dict(ORDERED_FREQUENCIES)
 """
 A dictionary with rotation frequency names (strings) as keys and
@@ -89,9 +108,6 @@
 filenames.
 """
 
-DEFAULT_REMOVAL_COMMAND = ['rm', '-fR']
-"""The default removal command (a list of strings)."""
-
 
 def coerce_location(value, **options):
     """
@@ -197,15 +213,21 @@ def load_config_file(configuration_file=None, expand=True):
         rotation_scheme = dict((name, coerce_retention_period(items[name]))
                                for name in SUPPORTED_FREQUENCIES
                                if name in items)
-        options = dict(include_list=split(items.get('include-list', '')),
-                       exclude_list=split(items.get('exclude-list', '')),
-                       io_scheduling_class=items.get('ionice'),
-                       strict=coerce_boolean(items.get('strict', 'yes')),
-                       prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
+        options = dict(
+            exclude_list=split(items.get('exclude-list', '')),
+            include_list=split(items.get('include-list', '')),
+            io_scheduling_class=items.get('ionice'),
+            prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')),
+            strict=coerce_boolean(items.get('strict', 'yes')),
+        )
         # Don't override the value of the 'removal_command' property unless the
         # 'removal-command' configuration file option has a value set.
         if items.get('removal-command'):
             options['removal_command'] = shlex.split(items['removal-command'])
+        # Don't override the value of the 'timestamp_pattern' property unless the
+        # 'timestamp-pattern' configuration file option has a value set.
+        if items.get('timestamp-pattern'):
+            options['timestamp_pattern'] = items['timestamp-pattern']
         # Expand filename patterns?
         if expand and location.have_wildcards:
             logger.verbose("Expanding filename pattern %s on %s ..", location.directory, location.context)
@@ -425,6 +447,40 @@ def strict(self):
         """
         return True
 
+    @mutable_property
+    def timestamp_pattern(self):
+        """
+        The pattern used to extract timestamps from filenames (defaults to :data:`TIMESTAMP_PATTERN`).
+
+        The value of this property is a compiled regular expression object.
+        Callers can provide their own compiled regular expression which
+        makes it possible to customize the compilation flags (see the
+        :func:`re.compile()` documentation for details).
+
+        The regular expression pattern is expected to be a Python compatible
+        regular expression that contains the named capture groups 'year',
+        'month', 'day', 'hour', 'minute' and 'second'.
+
+        String values are automatically coerced to compiled regular expressions
+        by calling :func:`~humanfriendly.coerce_pattern()`, in this case only
+        the :data:`re.VERBOSE` flag is used.
+
+        If the caller provides a custom pattern it will be validated
+        to confirm that the pattern contains named capture groups
+        corresponding to each of the required date components
+        defined by :data:`SUPPORTED_DATE_COMPONENTS`.
+        """
+        return TIMESTAMP_PATTERN
+
+    @timestamp_pattern.setter
+    def timestamp_pattern(self, value):
+        """Coerce the value of :attr:`timestamp_pattern` to a compiled regular expression."""
+        pattern = coerce_pattern(value, re.VERBOSE)
+        for component, required in SUPPORTED_DATE_COMPONENTS:
+            if component not in pattern.groupindex and required:
+                raise ValueError("Pattern is missing required capture group! (%s)" % component)
+        set_property(self, 'timestamp_pattern', pattern)
+
     def rotate_concurrent(self, *locations, **kw):
         """
         Rotate the backups in the given locations concurrently.
@@ -587,7 +643,7 @@ def collect_backups(self, location):
         logger.info("Scanning %s for backups ..", location)
         location.ensure_readable(self.force)
         for entry in natsort(location.context.list_entries(location.directory)):
-            match = TIMESTAMP_PATTERN.search(entry)
+            match = self.timestamp_pattern.search(entry)
             if match:
                 if self.exclude_list and any(fnmatch.fnmatch(entry, p) for p in self.exclude_list):
                     logger.verbose("Excluded %s (it matched the exclude list).", entry)
@@ -597,7 +653,7 @@ def collect_backups(self, location):
                     try:
                         backups.append(Backup(
                             pathname=os.path.join(location.directory, entry),
-                            timestamp=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),
+                            timestamp=self.match_to_datetime(match),
                         ))
                     except ValueError as e:
                         logger.notice("Ignoring %s due to invalid date (%s).", entry, e)
@@ -607,6 +663,30 @@ def collect_backups(self, location):
             logger.info("Found %i timestamped backups in %s.", len(backups), location)
         return sorted(backups)
 
+    def match_to_datetime(self, match):
+        """
+        Convert a regular expression match to a :class:`~datetime.datetime` value.
+
+        :param match: A regular expression match object.
+        :returns: A :class:`~datetime.datetime` value.
+        :raises: :exc:`exceptions.ValueError` when a required date component is
+                 not captured by the pattern, the captured value is an empty
+                 string or the captured value cannot be interpreted as a
+                 base-10 integer.
+
+        .. seealso:: :data:`SUPPORTED_DATE_COMPONENTS`
+        """
+        kw = {}
+        for component, required in SUPPORTED_DATE_COMPONENTS:
+            value = match.group(component)
+            if value:
+                kw[component] = int(value, 10)
+            elif required:
+                raise ValueError("Missing required date component! (%s)!" % component)
+            else:
+                kw[component] = 0
+        return datetime.datetime(**kw)
+
     def group_backups(self, backups):
         """
         Group backups collected by :func:`collect_backups()` by rotation frequencies.

rotate_backups/cli.py

@@ -1,7 +1,7 @@
 # rotate-backups: Simple command line interface for backup rotation.
 #
 # Author: Peter Odding <peter@peterodding.com>
-# Last Change: February 13, 2020
+# Last Change: February 14, 2020
 # URL: https://github.com/xolox/python-rotate-backups
 
 """
@@ -69,6 +69,13 @@
     Set the number of yearly backups to preserve during rotation. Refer to the
     usage of the -H, --hourly option for details about COUNT.
 
+  -t, --timestamp-pattern=PATTERN
+
+    Customize the regular expression pattern that is used to match and extract
+    timestamps from filenames. PATTERN is expected to be a Python compatible
+    regular expression that must define the named capture groups 'year',
+    'month' and 'day' and may define 'hour', 'minute' and 'second'.
+
   -I, --include=PATTERN
 
     Only process backups that match the shell pattern given by PATTERN. This
@@ -233,11 +240,12 @@ def main():
     selected_locations = []
     # Parse the command line arguments.
     try:
-        options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:C:uS:fnvqh', [
+        options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:t:I:x:jpri:c:C:uS:fnvqh', [
             'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=',
-            'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed',
-            'ionice=', 'config=', 'removal-command=', 'use-sudo', 'syslog=',
-            'force', 'dry-run', 'verbose', 'quiet', 'help',
+            'timestamp-pattern=', 'include=', 'exclude=', 'parallel',
+            'prefer-recent', 'relaxed', 'ionice=', 'config=',
+            'removal-command=', 'use-sudo', 'syslog=', 'force',
+            'dry-run', 'verbose', 'quiet', 'help',
         ])
         for option, value in options:
             if option in ('-M', '--minutely'):
@@ -252,6 +260,8 @@ def main():
                 rotation_scheme['monthly'] = coerce_retention_period(value)
             elif option in ('-y', '--yearly'):
                 rotation_scheme['yearly'] = coerce_retention_period(value)
+            elif option in ('-t', '--timestamp-pattern'):
+                kw['timestamp_pattern'] = value
             elif option in ('-I', '--include'):
                 kw['include_list'].append(value)
             elif option in ('-x', '--exclude'):

rotate_backups/tests.py

@@ -1,7 +1,7 @@
 # Test suite for the `rotate-backups' Python package.
 #
 # Author: Peter Odding <peter@peterodding.com>
-# Last Change: February 12, 2020
+# Last Change: February 14, 2020
 # URL: https://github.com/xolox/python-rotate-backups
 
 """Test suite for the `rotate-backups` package."""
@@ -455,6 +455,37 @@ def test_filename_patterns(self):
             assert any(location.directory == os.path.join(root, 'laptop') for location in available_locations)
             assert any(location.directory == os.path.join(root, 'vps') for location in available_locations)
 
+    def test_custom_timestamp_pattern(self):
+        """Test that custom timestamp patterns are properly supported."""
+        with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root:
+            custom_backup_filename = os.path.join(root, 'My-File--2009-12-31--23-59-59.txt')
+            touch(custom_backup_filename)
+            program = RotateBackups(
+                rotation_scheme=dict(monthly='always'),
+                timestamp_pattern=r'''
+                    (?P<year>\d{4}) - (?P<month>\d{2}) - (?P<day>\d{2})
+                    --
+                    (?P<hour>\d{2}) - (?P<minute>\d{2}) - (?P<second>\d{2})
+                ''',
+            )
+            backups = program.collect_backups(root)
+            assert backups[0].pathname == custom_backup_filename
+            assert backups[0].timestamp.year == 2009
+            assert backups[0].timestamp.month == 12
+            assert backups[0].timestamp.day == 31
+            assert backups[0].timestamp.hour == 23
+            assert backups[0].timestamp.minute == 59
+            assert backups[0].timestamp.second == 59
+
+    def test_invalid_timestamp_pattern(self):
+        """Test that the capture groups in custom timestamp patterns are validated."""
+        self.assertRaises(
+            ValueError,
+            RotateBackups,
+            rotation_scheme=dict(monthly='always'),
+            timestamp_pattern=r'(?P<year>\d{4})-(?P<month>\d{2})',
+        )
+
     def create_sample_backup_set(self, root):
         """Create a sample backup set to be rotated."""
         for name in SAMPLE_BACKUP_SET: