Basic rotation logic working

Author: timvaillancourt

mongodb_consistent_backup/Common/Config.py

@@ -73,6 +73,8 @@ def makeParser(self):
         parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)", default=None, type=str)
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
         parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
+        parser.add_argument("--rotate.max_backups", dest="rotate.max_backups", help="Maximum number of backups to keep in backup directory (default: disabled)", default=0, type=int)
+        parser.add_argument("--rotate.max_backup_days", dest="rotate.max_backup_days", help="Maximum age in days for backups in backup directory (default: disabled)", default=0, type=int)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)
         parser.add_argument("--sharding.balancer.ping_secs", dest="sharding.balancer.ping_secs", help="Interval to check balancer state, in seconds (default: 3)", default=3, type=int)
         return self.makeParserLoadSubmodules(parser)

mongodb_consistent_backup/Main.py

@@ -14,8 +14,9 @@
 from Notify import Notify
 from Oplog import Tailer, Resolver
 from Replication import Replset, ReplsetSharded
+from Rotate import Rotate
 from Sharding import Sharding
-from State import StateRoot, StateBackup, StateBackupReplset, StateDoneStamp, StateOplog
+from State import StateRoot, StateBackup, StateBackupReplset, StateOplog
 from Upload import Upload
 
 
@@ -97,8 +98,9 @@ def set_backup_dirs(self):
         self.backup_directory         = os.path.join(self.config.backup.location, self.backup_root_subdirectory)
 
     def setup_state(self):
-        StateRoot(self.backup_root_directory, self.config).write(True)
-        self.state = StateBackup(self.backup_directory, self.config, self.backup_time, self.uri, sys.argv)
+        self.state_root = StateRoot(self.backup_root_directory, self.config)
+        self.state      = StateBackup(self.backup_directory, self.config, self.backup_time, self.uri, sys.argv)
+        self.state_root.write(True)
         self.state.write()
 
     def setup_notifier(self):
@@ -167,6 +169,10 @@ def update_symlinks(self):
         logging.info("Updating %s latest symlink to: %s" % (self.config.backup.name, self.backup_directory))
         return os.symlink(self.backup_directory, self.backup_latest_symlink)
 
+    def rotate_backups(self):
+        rotater = Rotate(self.config, self.state_root)
+        rotater.run()
+
     # TODO Rename class to be more exact as this assumes something went wrong
     # noinspection PyUnusedLocal
     def cleanup_and_exit(self, code, frame):
@@ -451,6 +457,7 @@ def run(self):
 
         # stop timer
         self.stop_timer()
+        self.state.set("completed", True)
 
         # send notifications of backup state
         try:
@@ -466,8 +473,8 @@ def run(self):
             self.notify.close()
             self.exception("Problem running Notifier! Error: %s" % e, e)
 
-        StateDoneStamp(self.backup_directory, self.config).write()
         self.update_symlinks()
+        self.rotate_backups()
 
         self.logger.rotate()
         logging.info("Completed %s in %.2f sec" % (self.program_name, self.timer.duration(self.timer_name)))

mongodb_consistent_backup/State.py

@@ -37,10 +37,12 @@ def merge(self, new, old):
         merged.update(new)
         return merged
 
-    def load(self, load_one=False):
+    def load(self, load_one=False, filename=None):
         f = None
+        if not filename:
+            filename = self.state_file
         try:
-            f = open(self.state_file, "r")
+            f = open(filename, "r")
             data = decode_all(f.read())
             if load_one and len(data) > 0:
                 return data[0]
@@ -94,6 +96,7 @@ def __init__(self, base_dir, config, backup_time, seed_uri, argv=None):
         StateBase.__init__(self, base_dir, config)
         self.base_dir            = base_dir
         self.state['backup']     = True
+        self.state['completed']  = False
         self.state['name']       = backup_time
         self.state['method']     = config.backup.method
         self.state['path']       = base_dir
@@ -128,6 +131,8 @@ def __init__(self, base_dir, config):
         StateBase.__init__(self, base_dir, config)
         self.base_dir = base_dir
         self.state['root'] = True
+        self.backups = {}
+        self.completed_backups = 0
 
         self.init()
 
@@ -136,7 +141,6 @@ def init(self):
         self.load_backups()
 
     def load_backups(self):
-        backups = []
         if os.path.isdir(self.base_dir):
             for subdir in os.listdir(self.base_dir):
                 try:
@@ -145,16 +149,10 @@ def load_backups(self):
                         continue
                     state_path = os.path.join(bkp_path, self.meta_name)
                     state_file = os.path.join(state_path, "meta.bson")
-                    done_path  = os.path.join(state_path, "done.bson")
-                    if os.path.isdir(state_path) and os.path.isfile(state_file) and os.path.isfile(done_path):
-                        backups.append(state_file)
+                    self.backups[subdir] = self.load(True, state_file)
+                    if self.backups[subdir]["completed"]:
+                        self.completed_backups += 1
                 except:
                     continue
-            logging.info("Found %i existing completed backups for set" % len(backups))
-        return backups
-
-
-class StateDoneStamp(StateBase):
-    def __init__(self, base_dir, config):
-        StateBase.__init__(self, base_dir, config, "done.bson")
-        self.state = {'done': True}
+            logging.info("Found %i existing completed backups for set" % self.completed_backups)
+        return self.backups