Merge pull request #87 from timvaillancourt/MCB_1.0-cleanup_v4

timvaillancourt · web-flow · commit a9b008e58464 · 2017-01-25T15:48:19.000+01:00
Mcb 1.0 cleanup v4
diff --git a/mongodb_consistent_backup/Archive/Archive.py b/mongodb_consistent_backup/Archive/Archive.py
@@ -1,7 +1,7 @@
 import logging
 
 from Tar import Tar
-from mongodb_consistent_backup.Common import config_to_string, parse_submodule
+from mongodb_consistent_backup.Common import config_to_string, parse_method
 
 
 class Archive:
@@ -15,10 +15,10 @@ def __init__(self, config, backup_dir):
 
     def init(self):
         archive_method = self.config.archive.method
-        if not archive_method or parse_submodule(archive_method) == "none":
+        if not archive_method or parse_method(archive_method) == "none":
             logging.info("Archiving disabled, skipping")
         else:
-            self.method = parse_submodule(archive_method)
+            self.method = parse_method(archive_method)
             logging.info("Using archiving method: %s" % self.method)
             try:
                 self._archiver = globals()[self.method.capitalize()](
diff --git a/mongodb_consistent_backup/Archive/Tar/Tar.py b/mongodb_consistent_backup/Archive/Tar/Tar.py
@@ -6,6 +6,7 @@
 from types import MethodType
 
 from TarThread import TarThread
+from mongodb_consistent_backup.Common import parse_method
 
 
 # Allows pooled .apply_async()s to work on Class-methods:
@@ -28,9 +29,9 @@ def __init__(self, config, backup_base_dir):
 
     def compression(self, method=None):
         if method:
-            self.config.archive.tar.compression = method.lower()
+            self.config.archive.tar.compression = parse_method(method)
             logging.info("Setting tar compression method to: %s" % self.config.archive.tar.compression)
-        return self.config.archive.tar.compression
+        return parse_method(self.config.archive.tar.compression)
 
     def do_gzip(self):
         if self.compression() == 'gzip':
diff --git a/mongodb_consistent_backup/Backup/Backup.py b/mongodb_consistent_backup/Backup/Backup.py
@@ -1,7 +1,7 @@
 import logging
 
 from Mongodump import Mongodump
-from mongodb_consistent_backup.Common import config_to_string, parse_submodule
+from mongodb_consistent_backup.Common import config_to_string, parse_method
 
 
 class Backup:
@@ -17,9 +17,9 @@ def __init__(self, config, backup_dir, secondaries, config_server=None):
 
     def init(self):
         backup_method = self.config.backup.method
-        if not backup_method or parse_submodule(backup_method) == "none":
+        if not backup_method or parse_method(backup_method) == "none":
             raise Exception, 'Must specify a backup method!', None
-        self.method   = parse_submodule(backup_method)
+        self.method   = parse_method(backup_method)
         config_string = config_to_string(self.config.backup[self.method])
         logging.info("Using backup method: %s (options: %s)" % (self.method, config_string))
         try:
diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py
@@ -72,4 +72,5 @@ def connection(self):
         return self._conn
 
     def close(self):
-        return self._conn.close()
+        if self._conn:
+            return self._conn.close()
diff --git a/mongodb_consistent_backup/Common/Util.py b/mongodb_consistent_backup/Common/Util.py
@@ -1,13 +1,14 @@
 import socket
 
+
 def config_to_string(config):
     config_vars = ""
     for key in config:
         config_vars += "%s=%s," % (key, config[key])
     return config_vars[:-1]
 
-def parse_submodule(name):
-    return name.rstrip().lower()
+def parse_method(method):
+    return method.rstrip().lower()
 
 def validate_hostname(hostname):
     try:
diff --git a/mongodb_consistent_backup/Common/__init__.py b/mongodb_consistent_backup/Common/__init__.py
@@ -2,4 +2,4 @@
 from DB import DB
 from LocalCommand import LocalCommand
 from Lock import Lock
-from Util import config_to_string, parse_submodule, validate_hostname 
+from Util import config_to_string, parse_method, validate_hostname 
diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py
@@ -8,7 +8,7 @@
 
 from Archive import Archive
 from Backup import Backup
-from Common import DB, Lock, validate_hostname
+from Common import Config, DB, Lock, validate_hostname
 from Notify import Notify
 from Oplog import Tailer, Resolver
 from Replication import Replset, ReplsetSharded
@@ -17,38 +17,45 @@
 
 
 class MongodbConsistentBackup(object):
-    def __init__(self, config, prog_name="mongodb-consistent-backup"):
-        self.config          = config
-        self.program_name    = prog_name
-        self.backup          = None
-        self.archive         = None
-        self.sharding        = None
-        self.replset         = None
-        self.replset_sharded = None
-        self.notify          = None
-        self.oplogtailer     = None
-        self.oplog_resolver  = None
-        self.upload          = None
-        self.lock            = None
-        self.start_time      = time()
-        self.end_time        = None
-        self.backup_duration = None
-        self.backup_time = None
-        self.backup_root_directory = None
+    def __init__(self, prog_name="mongodb-consistent-backup"):
+        self.program_name             = prog_name
+        self.backup                   = None
+        self.archive                  = None
+        self.sharding                 = None
+        self.replset                  = None
+        self.replset_sharded          = None
+        self.notify                   = None
+        self.oplogtailer              = None
+        self.oplog_resolver           = None
+        self.upload                   = None
+        self.lock                     = None
+        self.start_time               = time()
+        self.end_time                 = None
+        self.backup_duration          = None
+        self.backup_time              = None
+        self.backup_root_directory    = None
         self.backup_root_subdirectory = None
-        self.connection      = None
-        self.db              = None
-        self.is_sharded      = False
-        self.secondaries     = {}
-        self.oplog_summary   = {}
-        self.backup_summary  = {}
-        self.log_level = None
-
+        self.connection               = None
+        self.db                       = None
+        self.is_sharded               = False
+        self.secondaries              = {}
+        self.oplog_summary            = {}
+        self.backup_summary           = {}
+        self.log_level                = None
+
+        self.setup_config()
         self.setup_signal_handlers()
         self.setup_logger()
         self.set_backup_dirs()
         self.get_db_conn()
 
+    def setup_config(self):
+        try:
+            self.config = Config()
+        except Exception, e:
+            print "Error setting up configuration: '%s'!" % e
+            sys.exit(1)
+
     def setup_logger(self):
         self.log_level = logging.INFO
         if self.config.verbose:
@@ -150,7 +157,7 @@ def run(self):
         try:
             self.notify = Notify(self.config)
         except Exception, e:
-            raise e
+            self.exception("Problem starting notifier! Error: %s" % e)
 
         # Setup the archiver
         try:
@@ -159,7 +166,17 @@ def run(self):
                 self.backup_root_directory, 
             )
         except Exception, e:
-            raise e
+            self.exception("Problem starting archiver! Error: %s" % e)
+
+        # Setup the uploader
+        try:
+            self.upload = Upload(
+                self.config,
+                self.backup_root_directory,
+                self.backup_root_subdirectory
+            )
+        except Exception, e:
+            self.exception("Problem starting uploader! Error: %s" % e)
 
         if not self.is_sharded:
             logging.info("Running backup of %s:%s in replset mode" % (self.config.host, self.config.port))
@@ -286,11 +303,6 @@ def run(self):
 
         # upload backup
         try:
-            self.upload = Upload(
-                self.config,
-                self.backup_root_directory,
-                self.backup_root_subdirectory
-            )
             self.upload.upload()
         except Exception, e:
             self.exception("Problem performing upload of backup! Error: %s" % e)
diff --git a/mongodb_consistent_backup/Notify/Notify.py b/mongodb_consistent_backup/Notify/Notify.py
@@ -1,6 +1,6 @@
 import logging
 
-from mongodb_consistent_backup.Common import config_to_string, parse_submodule
+from mongodb_consistent_backup.Common import config_to_string, parse_method
 
 
 class Notify:
@@ -13,10 +13,10 @@ def __init__(self, config):
 
     def init(self):
         notify_method = self.config.notify.method
-        if not notify_method or parse_submodule(notify_method) == "none":
+        if not notify_method or parse_method(notify_method) == "none":
             logging.info("Notifying disabled, skipping")
         else:
-            self.method   = parse_submodule(notify_method)
+            self.method   = parse_method(notify_method)
             config_string = config_to_string(self.config.notify[self.method])
             logging.info("Using notify method: %s (options: %s)" % (self.method, config_string))
             try:
diff --git a/mongodb_consistent_backup/Oplog/Tailer/Tailer.py b/mongodb_consistent_backup/Oplog/Tailer/Tailer.py
@@ -4,6 +4,7 @@
 from time import sleep
 
 from TailerThread import TailerThread
+from mongodb_consistent_backup.Common import parse_method
 
 
 class Tailer:
@@ -22,9 +23,9 @@ def __init__(self, config, secondaries, base_dir):
 
     def compression(self, method=None):
         if method:
-            self.config.oplog.compression = method.lower()
+            self.config.oplog.compression = parse_method(method)
             logging.info("Setting oplog compression method to: %s" % self.config.oplog.compression)
-        return self.config.oplog.compression
+        return parse_method(self.config.oplog.compression)
 
     def do_gzip(self):
         if self.compression() == 'gzip':
diff --git a/mongodb_consistent_backup/Upload/Upload.py b/mongodb_consistent_backup/Upload/Upload.py
@@ -1,7 +1,7 @@
 import logging
 
 from S3 import S3
-from mongodb_consistent_backup.Common import config_to_string, parse_submodule
+from mongodb_consistent_backup.Common import config_to_string, parse_method
 
 
 class Upload:
@@ -16,10 +16,10 @@ def __init__(self, config, base_dir, backup_dir):
 
     def init(self):
         upload_method = self.config.upload.method
-        if not upload_method or parse_submodule(upload_method) == "none":
+        if not upload_method or parse_method(upload_method) == "none":
             logging.info("Uploading disabled, skipping")
         else:
-            self.method   = parse_submodule(upload_method)
+            self.method   = parse_method(upload_method)
             config_string = config_to_string(self.config.upload[self.method])
             logging.info("Using upload method: %s (options: %s)" % (self.method, config_string))
             try:
diff --git a/mongodb_consistent_backup/__init__.py b/mongodb_consistent_backup/__init__.py
@@ -1,24 +1,12 @@
-# TODO-timv Can we remove this?
-import os
 import sys
 
-from Common import Config
 from Main import MongodbConsistentBackup
 
-__version__ = '#.#.#'
-git_commit  = 'GIT_COMMIT_HASH'
-
 
 # noinspection PyUnusedLocal
 def run():
     try:
-        config = Config()
-    except Exception, e:
-        print "Error setting up configuration: '%s'!" % e
-        sys.exit(1)
-
-    try:
-        MongodbConsistentBackup(config).run()
+        MongodbConsistentBackup().run()
     except Exception, e:
         # noinspection PyUnusedLocal
         print "Backup '%s' failed for mongodb instance %s:%s : %s" % (config.name, config.host, config.port, e)
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -66,7 +66,6 @@ if [ -d ${srcdir} ]; then
 			exit 1
 		else
 			sed -i -e s@\#.\#.\#@${version}@g ${builddir}/setup.py
-			sed -i -e s@\#.\#.\#@${version}@g ${builddir}/${mod_name}/__init__.py
 			sed -i -e s@\#.\#.\#@${version}@g ${builddir}/${mod_name}/Common/Config.py
 		fi
 	else
@@ -78,7 +77,6 @@ if [ -d ${srcdir} ]; then
 	if [ -z "$git_commit" ]; then
 		echo "Warning: cannot find git commit hash!"
 	else
-		sed -i -e s@GIT_COMMIT_HASH@${git_commit}@g ${builddir}/${mod_name}/__init__.py
 		sed -i -e s@GIT_COMMIT_HASH@${git_commit}@g ${builddir}/${mod_name}/Common/Config.py
 	fi
 
@@ -99,7 +97,7 @@ if [ -d ${srcdir} ]; then
 	if [ ! -d ${pexdir} ]; then
 		mkdir -p ${pexdir}
 	else
-		rm -f ${pexdir}/build/mongodb_consistent_backup-* 
+		rm -f ${pexdir}/build/mongodb_consistent_backup-*.whl 
 	fi
 	[ ! -d ${bindir} ] && mkdir -p ${bindir}
 	${venvdir}/bin/python2.7 ${venvdir}/bin/pex -o ${output_file} -m ${mod_name} -r ${require_file} --pex-root=${pexdir}  ${builddir}
