Merge pull request #101 from timvaillancourt/MCB_1.0-bugfix12

MCB_1.0: Bugfixes v12

Author: timvaillancourt

mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py

@@ -7,6 +7,7 @@
 from signal import signal, SIGINT, SIGTERM, SIG_IGN
 from subprocess import Popen, PIPE
 
+from mongodb_consistent_backup.Common import is_datetime
 from mongodb_consistent_backup.Oplog import Oplog
 
 
@@ -52,6 +53,8 @@ def parse_mongodump_line(self, line):
                 return None
             if "\t" in line:
                 (date, line) = line.split("\t")
+            elif is_datetime(line):
+                return None
             return "%s:\t%s" % (self.uri, line) 
         except:
             return None

mongodb_consistent_backup/Common/Util.py

@@ -1,5 +1,7 @@
 import socket
 
+from dateutil import parser
+
 from mongodb_consistent_backup.Errors import OperationError
 
 
@@ -9,6 +11,13 @@ def config_to_string(config):
         config_vars += "%s=%s, " % (key, config[key])
     return config_vars[:-1]
 
+def is_datetime(string):
+    try:
+        parser.parse(string)
+        return True
+    except:
+        return False
+
 def parse_method(method):
     return method.rstrip().lower()
 

mongodb_consistent_backup/Common/__init__.py

@@ -4,4 +4,4 @@
 from Lock import Lock
 from MongoUri import MongoUri
 from Timer import Timer
-from Util import config_to_string, parse_method, validate_hostname 
+from Util import config_to_string, is_datetime, parse_method, validate_hostname 

mongodb_consistent_backup/Logger.py

@@ -28,17 +28,23 @@ def __init__(self, config, backup_time):
     def start(self):
         try:
             logging.basicConfig(level=self.log_level, format=self.log_format)
-            if self.do_file_log:
+        except Exception, e:
+            print("Could not start logger: %s" % e)
+            raise e
+
+    def start_file_logger(self):
+        if self.do_file_log:
+            try:
                 self.current_log_file = os.path.join(self.config.log_dir, "backup.%s.log" % self.backup_name)
                 self.backup_log_file  = os.path.join(self.config.log_dir, "backup.%s.%s.log" % (self.backup_name, self.backup_time))
                 self.file_log = logging.FileHandler(self.backup_log_file)
                 self.file_log.setLevel(self.log_level)
                 self.file_log.setFormatter(logging.Formatter(self.log_format))
                 logging.getLogger('').addHandler(self.file_log)
                 self.update_symlink()
-        except OSError, e:
-            logging.warning("Could not start file log handler, writing to stdout only")
-            pass
+            except OSError, e:
+                logging.warning("Could not start file log handler, writing to stdout only")
+                pass
 
     def close(self):
         if self.file_log:
@@ -61,6 +67,8 @@ def compress(self):
                 gz_log.close()
 
     def update_symlink(self):
+        if not self.do_file_log:
+            return
         if os.path.islink(self.current_log_file):
             self.last_log = os.readlink(self.current_log_file)
             os.remove(self.current_log_file)

mongodb_consistent_backup/Main.py

@@ -55,6 +55,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
             self.setup_logger()
             self.setup_signal_handlers()
             self.get_lock()
+            self.logger.start_file_logger()
             self.init()
             self.set_backup_dirs()
             self.get_db_conn()
@@ -116,7 +117,7 @@ def get_lock(self):
             self.lock = Lock(self.config.lock_file)
         except Exception:
             logging.fatal("Could not acquire lock: '%s'! Is another %s process running? Exiting" % (self.config.lock_file, self.program_name))
-            self.cleanup_and_exit(None, None)
+            sys.exit(1)
 
     def release_lock(self):
         if self.lock:

mongodb_consistent_backup/State.py

@@ -9,14 +9,14 @@
 
 
 class StateBase(object):
-    def __init__(self, base_dir, config, filename="meta.bson", state_version=1):
-        self.state_dir  = os.path.join(base_dir, "mongodb-consistent-backup_META")
-        self.state_lock = os.path.join(base_dir, "mongodb-consistent-backup_META.lock")
+    def __init__(self, base_dir, config, filename="meta.bson", state_version=1, meta_name="mongodb-consistent-backup_META"):
+        self.meta_name  = meta_name
+        self.state_dir  = os.path.join(base_dir, self.meta_name)
+        self.state_lock = os.path.join(base_dir, "%s.lock" % self.meta_name)
         self.state_file = os.path.join(self.state_dir, filename)
         self.state = {
             "name":          config.backup.name,
             "path":          base_dir,
-            "hostname":      platform.node(),
             "state_version": state_version
         }
         self.lock = Lock(self.state_lock, False)
@@ -129,10 +129,10 @@ def load_backups(self):
         if os.path.isdir(self.base_dir):
             for subdir in os.listdir(self.base_dir):
                 try:
-                    if subdir == "mongodb-consistent-backup_META" :
+                    if subdir == self.meta_name:
                         continue
                     bkp_path   = os.path.join(self.base_dir, subdir)
-                    state_path = os.path.join(bkp_path, "mongodb-consistent-backup_META")
+                    state_path = os.path.join(bkp_path, self.meta_name)
                     state_file = os.path.join(state_path, "meta.bson")
                     done_path  = os.path.join(state_path, "done.bson")
                     if os.path.isdir(state_path) and os.path.isfile(state_file) and os.path.isfile(done_path):

requirements.txt

@@ -6,3 +6,4 @@ PyYAML==3.12
 boto==2.45.0
 filechunkio==1.8
 yconf==0.3.3
+python-dateutil==2.2