Merge pull request #208 from timvaillancourt/1.2.0-ssl-pymongo

1.2.0: MongoDB SSL support (#60)

Author: dbmurphy

README.rst

@@ -35,6 +35,7 @@ Features
 -  `Nagios NSCA <https://sourceforge.net/p/nagios/nsca>`__ push
    notification support (*optional*)
 -  Modular backup, archiving, upload and notification components
+-  Support for MongoDB Authentication and SSL database connections
 -  Multi-threaded, single executable
 -  Auto-scales to number of available CPUs by default
 
@@ -221,6 +222,7 @@ Roadmap
 -  Upload compatibility for ZBackup archive phase *(upload unsupported today)*
 -  Backup retention/rotation *(eg: delete old backups)*
 -  Support more notification methods *(Prometheus, PagerDuty, etc)*
+-  Support more upload methods *(Rsync, etc)*
 -  Support SSL MongoDB connections
 -  Documentation for running under Docker with persistent volumes
 -  Python unit tests

conf/mongodb-consistent-backup.example.conf

@@ -4,6 +4,12 @@ production:
   #username: [auth username] (default: none)
   #password: [auth password] (default: none)
   #authdb: [auth database]   (default: admin)
+  #ssl:
+  #  enabled: [true|false]  (default: false)
+  #  insecure: [true|false] (default: false)
+  #  ca_file: [path]
+  #  crl_file: [path]
+  #  client_cert_file: [path]
   log_dir: /var/log/mongodb-consistent-backup
   backup:
     method: mongodump

mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py

@@ -8,7 +8,7 @@
 from signal import signal, SIGINT, SIGTERM, SIG_IGN
 from subprocess import Popen, PIPE
 
-from mongodb_consistent_backup.Common import is_datetime
+from mongodb_consistent_backup.Common import is_datetime, parse_config_bool
 from mongodb_consistent_backup.Oplog import Oplog
 
 
@@ -25,10 +25,13 @@ def __init__(self, state, uri, timer, config, base_dir, version, threads=0, dump
         self.threads   = threads
         self.dump_gzip = dump_gzip
 
-        self.user     = self.config.username
-        self.password = self.config.password
-        self.authdb   = self.config.authdb
-        self.binary   = self.config.backup.mongodump.binary
+        self.user                 = self.config.username
+        self.password             = self.config.password
+        self.authdb               = self.config.authdb
+        self.ssl_ca_file          = self.config.ssl.ca_file
+        self.ssl_crl_file         = self.config.ssl.crl_file
+        self.ssl_client_cert_file = self.config.ssl.client_cert_file
+        self.binary               = self.config.backup.mongodump.binary
 
         self.timer_name        = "%s-%s" % (self.__class__.__name__, self.uri.replset)
         self.exit_code         = 1
@@ -52,6 +55,18 @@ def close(self, exit_code=None, frame=None):
             self._command.close()
         sys.exit(self.exit_code)
 
+    def do_ssl(self):
+        return parse_config_bool(self.config.ssl.enabled)
+
+    def do_ssl_insecure(self):
+        return parse_config_bool(self.config.ssl.insecure)
+
+    def is_version_gte(self, compare):
+        if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK):
+            if tuple(compare.split(".")) <= tuple(self.version.split(".")):
+                return True
+        return False
+
     def parse_mongodump_line(self, line):
         try:
             line = line.rstrip()
@@ -118,21 +133,40 @@ def mongodump_cmd(self):
         mongodump_flags = ["--host", mongodump_uri.host, "--port", str(mongodump_uri.port), "--oplog", "--out", "%s/dump" % self.backup_dir]
         if self.threads > 0:
             mongodump_flags.extend(["--numParallelCollections=" + str(self.threads)])
+
         if self.dump_gzip:
             mongodump_flags.extend(["--gzip"])
-        if tuple("3.4.0".split(".")) <= tuple(self.version.split(".")):
+
+        if self.is_version_gte("3.4.0"):
             mongodump_flags.extend(["--readPreference=secondary"])
+
         if self.authdb and self.authdb != "admin":
             logging.debug("Using database %s for authentication" % self.authdb)
             mongodump_flags.extend(["--authenticationDatabase", self.authdb])
         if self.user and self.password:
             # >= 3.0.2 supports password input via stdin to mask from ps
-            if tuple(self.version.split(".")) >= tuple("3.0.2".split(".")):
+            if self.is_version_gte("3.0.2"):
                 mongodump_flags.extend(["-u", self.user, "-p", '""'])
                 self.do_stdin_passwd = True
             else:
                 logging.warning("Mongodump is too old to set password securely! Upgrade to mongodump >= 3.0.2 to resolve this")
                 mongodump_flags.extend(["-u", self.user, "-p", self.password])
+
+        if self.do_ssl():
+            if self.is_version_gte("2.6.0"):
+                mongodump_flags.append("--ssl")
+                if self.ssl_ca_file:
+                    mongodump_flags.extend(["--sslCAFile", self.ssl_ca_file])
+                if self.ssl_crl_file:
+                    mongodump_flags.extend(["--sslCRLFile", self.ssl_crl_file])
+                if self.client_cert_file:
+                    mongodump_flags.extend(["--sslPEMKeyFile", self.ssl_cert_file])
+                if self.do_ssl_insecure():
+                    mongodump_flags.extend(["--sslAllowInvalidCertificates", "--sslAllowInvalidHostnames"])
+            else:
+                logging.fatal("Mongodump must be >= 2.6.0 to enable SSL encryption!")
+                sys.exit(1)
+
         mongodump_cmd.extend(mongodump_flags)
         return mongodump_cmd
 

mongodb_consistent_backup/Common/Config.py

@@ -8,6 +8,18 @@
 from yconf.util import NestedDict
 
 
+def parse_config_bool(item):
+    try:
+        if isinstance(item, bool):
+            return item
+        elif isinstance(item, str):
+            if item.rstrip().lower() is "true":
+                return True
+        return False
+    except:
+        return False
+
+
 class PrintVersions(Action):
     def __init__(self, option_strings, dest, nargs=0, **kwargs):
         super(PrintVersions, self).__init__(option_strings=option_strings, dest=dest, nargs=nargs, **kwargs)
@@ -54,6 +66,11 @@ def makeParser(self):
         parser.add_argument("-u", "--user", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
+        parser.add_argument("--ssl.enabled", dest="ssl.enabled", help="Use SSL secured database connections to MongoDB hosts (default: false)", default=False, action="store_true")
+        parser.add_argument("--ssl.insecure", dest="ssl.insecure", help="Do not validate the SSL certificate and hostname of the server (default: false)", default=False, action="store_true")
+        parser.add_argument("--ssl.ca_file", dest="ssl.ca_file", help="Path to SSL Certificate Authority file in PEM format (default: use OS default CA)", default=None, type=str)
+        parser.add_argument("--ssl.crl_file", dest="ssl.crl_file", help="Path to SSL Certificate Revocation List file in PEM or DER format (for optional cert revocation)", default=None, type=str)
+        parser.add_argument("--ssl.client_cert_file", dest="ssl.client_cert_file", help="Path to Client SSL Certificate file in PEM format (for optional client ssl auth)", default=None, type=str)
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
         parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)

mongodb_consistent_backup/Common/DB.py

@@ -4,46 +4,85 @@
 from inspect import currentframe, getframeinfo
 from pymongo import DESCENDING, CursorType, MongoClient
 from pymongo.errors import ConnectionFailure, OperationFailure, ServerSelectionTimeoutError
+from ssl import CERT_REQUIRED, CERT_NONE
 from time import sleep
 
+from mongodb_consistent_backup.Common import parse_config_bool
 from mongodb_consistent_backup.Errors import DBAuthenticationError, DBConnectionError, DBOperationError, Error
 
 
 class DB:
     def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_connect=True, conn_timeout=5000, retries=5):
         self.uri          = uri
-        self.username     = config.username
-        self.password     = config.password
-        self.authdb       = config.authdb
+        self.config       = config
         self.do_replset   = do_replset
         self.read_pref    = read_pref
         self.do_connect   = do_connect
         self.conn_timeout = conn_timeout
         self.retries      = retries
 
+        self.username             = self.config.username
+        self.password             = self.config.password
+        self.authdb               = self.config.authdb
+        self.ssl_ca_file          = self.config.ssl.ca_file
+        self.ssl_crl_file         = self.config.ssl.crl_file
+        self.ssl_client_cert_file = self.config.ssl.client_cert_file
+
         self.replset    = None
         self._conn      = None
         self._is_master = None
+
         self.connect()
         self.auth_if_required()
 
+    def do_ssl(self):
+        return parse_config_bool(self.config.ssl.enabled)
+
+    def do_ssl_insecure(self):
+        return parse_config_bool(self.config.ssl.insecure)
+
+    def client_opts(self):
+        opts = {
+            "connect":                  self.do_connect,
+            "host":                     self.uri.hosts(),
+            "connectTimeoutMS":         self.conn_timeout,
+            "serverSelectionTimeoutMS": self.conn_timeout,
+            "maxPoolSize":              1,
+        }
+        if self.do_replset:
+            self.replset = self.uri.replset
+            opts.update({
+                "replicaSet":     self.replset,
+                "readPreference": self.read_pref,
+                "w":              "majority"
+            })
+        if self.do_ssl():
+            logging.debug("Using SSL-secured mongodb connection (ca_cert=%s, client_cert=%s, crl_file=%s, insecure=%s)" % (
+                self.ssl_ca_file,
+                self.ssl_client_cert_file,
+                self.ssl_crl_file,
+                self.do_ssl_insecure()
+            ))
+            opts.update({
+                "ssl":           True,
+                "ssl_ca_certs":  self.ssl_ca_file,
+                "ssl_crlfile":   self.ssl_crl_file,
+                "ssl_certfile":  self.ssl_client_cert_file,
+                "ssl_cert_reqs": CERT_REQUIRED,
+            })
+            if self.do_ssl_insecure():
+                opts["ssl_cert_reqs"] = CERT_NONE
+        return opts
+
     def connect(self):
         try:
-            if self.do_replset:
-                self.replset = self.uri.replset
-            logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s)" % (
-                self.uri, self.replset, self.read_pref
+            logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, ssl=%s)" % (
+                self.uri,
+                self.replset,
+                self.read_pref,
+                self.do_ssl(),
             ))
-            conn = MongoClient(
-                connect=self.do_connect,
-                host=self.uri.hosts(),
-                replicaSet=self.replset,
-                readPreference=self.read_pref,
-                connectTimeoutMS=self.conn_timeout,
-                serverSelectionTimeoutMS=self.conn_timeout,
-                maxPoolSize=1,
-                w="majority"
-            )
+            conn = MongoClient(**self.client_opts())
             if self.do_connect:
                 conn['admin'].command({"ping": 1})
         except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e:

mongodb_consistent_backup/Common/__init__.py

@@ -1,4 +1,4 @@
-from Config import Config  # NOQA
+from Config import Config, parse_config_bool  # NOQA
 from DB import DB  # NOQA
 from LocalCommand import LocalCommand  # NOQA
 from Lock import Lock  # NOQA