Merge branch 'master' into 1.2.0-read_pref_tags

Author: timvaillancourt

README.rst

@@ -31,9 +31,11 @@ Features
    archiving method (*optional*)
 -  `AWS S3 <https://aws.amazon.com/s3/>`__ Secure Multipart backup uploads (*optional*)
 -  `Google Cloud Storage <https://cloud.google.com/storage/>`__ Secure backup uploads (*optional*)
+-  Rsync (over SSH) secure backup uploads (*optional*)
 -  `Nagios NSCA <https://sourceforge.net/p/nagios/nsca>`__ push
    notification support (*optional*)
 -  Modular backup, archiving, upload and notification components
+-  Support for MongoDB Authentication and SSL database connections
 -  Multi-threaded, single executable
 -  Auto-scales to number of available CPUs by default
 
@@ -77,6 +79,7 @@ To build an CentOS/RedHat RPM of the tool *(recommended)*:
 ::
 
     $ cd /path/to/mongodb_consistent_backup
+    $ sudo yum install -y rpm-build
     $ make rpm
 
 To build and install from source *(to default '/usr/local/bin/mongodb-consistent-backup')*:
@@ -148,25 +151,40 @@ The backups are `mongorestore <https://docs.mongodb.com/manual/reference/program
     ...
     $ mongorestore --host mongod12.example.com --port 27017 -u admin -p 123456 --oplogReplay --dir /var/lib/mongodb-consistent-backup/default/20170424_0000/rs0/dump
 
-Run as Docker Container (Experimental)
+Run as Docker Container
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Note: you need to use persistent volumes to store backups and/or config files long-term when using Docker. Data in Docker containers is destroyed when the container is deleted. See `scripts/docker-persistent.sh <scripts/docker-persistent.sh>`__ and `scripts/docker-persistent.example.conf <scripts/docker-persistent.example.conf>`__ as an example/demo of how to implement persistence.
+To persist logs, configs and backup data 3 directories should be mapped to be inside the Docker containter.
+
+The 'docker run' command -v/--volume flags in the examples below map container paths to paths on your Docker host. The example below assumes there is a path on the Docker host named *'/data/mongobackup'* with *'data'*, *'conf'* and *'logs'* subdirectories mapped to inside the container. Replace any instance of *'/data/mongobackup'* below to a different path if necessary.
+
+*Note: store a copy of your mongodb-consistent-backup.conf in the 'conf' directory and pass it's container path as the --config= flag if you wish to use config files.*
 
 **Via Docker Hub**
 
 ::
 
-    $ docker run -i timvaillancourt/mongodb_consistent_backup <mongodb_consistent_backup-flags>
+    $ mkdir -p /data/mongobackup/{conf,data,logs}
+    $ cp -f /path/to/mongodb-consistent-backup.conf /data/mongobackup/conf
+    $ docker run -it \
+        -v "/data/mongobackup/conf:/conf:Z" \
+        -v "/data/mongobackup/data:/var/lib/mongodb-consistent-backup:Z" \
+        -v "/data/mongobackup/logs:/var/log/mongodb-consistent-backup:Z" \
+      timvaillancourt/mongodb_consistent_backup:latest --config=/conf/mongodb-consistent-backup.conf
 
 **Build and Run Docker Image**
 
 ::
 
     $ cd /path/to/mongodb_consistent_backup
     $ make docker
-    $ docker run -t mongodb_consistent_backup <mongodb_consistent_backup-flags>
-
+    $ mkdir -p /data/mongobackup/{conf,data,logs}
+    $ cp -f /path/to/mongodb-consistent-backup.conf /data/mongobackup/conf
+    $ docker run -it \
+        -v "/data/mongobackup/conf:/conf:Z" \
+        -v "/data/mongobackup/data:/var/lib/mongodb-consistent-backup:Z" \
+        -v "/data/mongobackup/logs:/var/log/mongodb-consistent-backup:Z" \
+      mongodb_consistent_backup --config=/conf/mongodb-consistent-backup.conf
 
 ZBackup Archiving (Optional)
 ~~~~~~~

VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

conf/mongodb-consistent-backup.example.conf

@@ -17,8 +17,8 @@ production:
     location: /var/lib/mongodb-consistent-backup
   #  mongodump:
   #    binary: [path]                (default: /usr/bin/mongodump)
-  #    compression: [auto|none|gzip] (default: auto - enable gzip if supported)
-  #    threads: [1-16]               (default: auto-generated - shards/cpu)
+  #    compression: [auto|none|gzip] (default: auto = enable gzip if supported)
+  #    threads: [1-16]               (default: auto-generated, shards/cpu)
   #replication:
   #  max_lag_secs: [1+]                        (default: 10)
   #  min_priority: [0-999]                     (default: 0)
@@ -30,18 +30,19 @@ production:
   #    wait_secs: [1+] (default: 300)
   #    ping_secs: [1+] (default: 3)
   #oplog:
-  #  compression: [none|gzip] (default: gzip - if gzip is used by backup stage)
+  #  compression: [none|gzip] (default: gzip, if used by backup stage)
   #  flush:
-  #    max_docs: 100
-  #    max_secs: 1
-  #  resolver_threads: [1+]   (default: 2 per CPU)
+  #    max_docs: [1+] (default: 100)
+  #    max_secs: [1+] (default: 1)
+  #  resolver:
+  #    threads: [1+]  (default: 2 per CPU)
   #  tailer:
   #    enabled: true
   #    status_interval: 30
   archive:
     method: tar
   #  tar:
-  #    compression: [none|gzip] (default: gzip - none if backup is already compressed)
+  #    compression: [none|gzip] (default: gzip, none if backup already compressed)
   #    threads: [1+]            (default: 1 per CPU)
   #  zbackup:
   #    binary: [path]        (default: /usr/bin/zbackup)
@@ -59,21 +60,26 @@ production:
   upload:
     method: none
   #  remove_uploaded: [true|false] (default: false)
+  #  retries: [1+]           (default: 5)
+  #  threads: [1+]           (default: 4)
   #  gs:
   #    project_id: [Google Cloud Project ID]
-  #    access_key: [Google Cloud Storage Access Key]
-  #    secret_key: [Google Cloud Storage Secret Key]
+  #    access_key: [Google Cloud Storage Interoperability API Access Key]
+  #    secret_key: [Google Cloud Storage Interoperability API Secret Key]
   #    bucket_name: [Google Cloud Storage Bucket Name]
   #    bucket_prefix: [prefix] (default: /)
-  #    threads: [1+]           (default: 1 per CPU)
+  #  rsync:
+  #    path: [Rsync Destination Path]
+  #    user: [SSH Username]
+  #    host: [SSH Hostname/IP]
+  #    port: [SSH Port Number] (default: 22)
+  #    delete: [true|false]    (default: false)
   #  s3:
   #    region: [AWS S3 Region] (default: us-east-1)
   #    access_key: [AWS S3 Access Key]
   #    secret_key: [AWS S3 Secret Key]
   #    bucket_name: [AWS S3 Bucket Name]
   #    bucket_prefix: [prefix] (default: /)
-  #    threads: [1+]           (default: 1 per CPU)
   #    chunk_size_mb: [1+]     (default: 50)
   #    secure: [true|false]    (default: true)
-  #    retries: [1+]           (default: 5)
   #    acl: [acl_str]          (default: none)

mongodb_consistent_backup/Backup/Mongodump/Mongodump.py

@@ -1,6 +1,5 @@
 import os
 import logging
-import sys
 
 from math import floor
 from subprocess import check_output
@@ -57,22 +56,21 @@ def parse_mongodump_version(self):
         raise OperationError("Could not parse mongodump --version output!")
 
     def choose_compression(self):
-        if self.can_gzip():
+        if self.can_compress():
             if self.compression() == 'auto':
                 logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression")
                 self.compression('gzip')
         elif self.compression() == 'gzip':
             raise OperationError("mongodump gzip compression requested on binary that does not support gzip!")
 
-    def can_gzip(self):
+    def can_compress(self):
         if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK):
-            logging.debug("Mongodump binary supports gzip")
+            logging.debug("Mongodump binary supports gzip compression")
             if tuple("3.2.0".split(".")) <= tuple(self.version.split(".")):
                 return True
             return False
         else:
-            logging.fatal("Cannot find or execute the mongodump binary file %s!" % self.binary)
-            sys.exit(1)
+            raise OperationError("Cannot find or execute the mongodump binary file %s!" % self.binary)
 
     def summary(self):
         return self._summary

mongodb_consistent_backup/Common/Config.py

@@ -105,7 +105,13 @@ def check_required(self):
             try:
                 self._get(key)
             except:
-                raise mongodb_consistent_backup.Errors.OperationError('Field "%s" must be set via command-line or config file!' % key)
+                raise mongodb_consistent_backup.Errors.OperationError(
+                    'Field "%s" (config file field: "%s.%s") must be set via command-line or config file!' % (
+                        key,
+                        self._config.environment,
+                        key
+                    )
+                )
 
     def parse(self):
         self._config.parse(self.cmdline)

mongodb_consistent_backup/Common/DB.py

@@ -40,6 +40,13 @@ def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred',
         self.ssl_client_cert_file = self.config.ssl.client_cert_file
         self.read_pref_tags       = self.config.replication.read_pref_tags
 
+        self.username             = self.config.username
+        self.password             = self.config.password
+        self.authdb               = self.config.authdb
+        self.ssl_ca_file          = self.config.ssl.ca_file
+        self.ssl_crl_file         = self.config.ssl.crl_file
+        self.ssl_client_cert_file = self.config.ssl.client_cert_file
+
         self.replset    = None
         self._conn      = None
         self._is_master = None

mongodb_consistent_backup/Common/Util.py

@@ -1,6 +1,7 @@
 import socket
 
 from dateutil import parser
+from select import select
 
 from mongodb_consistent_backup.Errors import OperationError
 
@@ -31,3 +32,23 @@ def validate_hostname(hostname):
         socket.getaddrinfo(hostname, None)
     except socket.error, e:
         raise OperationError("Could not resolve host '%s', error: %s" % (hostname, e))
+
+
+def wait_popen(process, stderr_callback, stdout_callback):
+    try:
+        while not process.returncode:
+            poll = select([process.stderr.fileno(), process.stdout.fileno()], [], [])
+            if len(poll) >= 1:
+                for fd in poll[0]:
+                    if process.stderr and fd == process.stderr.fileno():
+                        stderr_callback(process.stderr.readline().rstrip())
+                    if process.stdout and fd == process.stdout.fileno():
+                        stdout_callback(process.stdout.readline().rstrip())
+            if process.poll() is not None:
+                break
+        stderr, stdout = process.communicate()
+        stderr_callback(stderr.rstrip())
+        stdout_callback(stdout.rstrip())
+    except Exception, e:
+        raise e
+    return True

mongodb_consistent_backup/Common/__init__.py

@@ -4,4 +4,4 @@
 from Lock import Lock  # NOQA
 from MongoUri import MongoUri  # NOQA
 from Timer import Timer  # NOQA
-from Util import config_to_string, is_datetime, parse_method, validate_hostname  # NOQA
+from Util import config_to_string, is_datetime, parse_method, validate_hostname, wait_popen  # NOQA

mongodb_consistent_backup/Pipeline/Stage.py

@@ -64,6 +64,7 @@ def close(self):
     def is_compressed(self):
         if self.has_task() and hasattr(self._task, "is_compressed"):
             return self._task.is_compressed()
+        return False
 
     def compression(self, task=None):
         if self.has_task() and hasattr(self._task, "compression"):

mongodb_consistent_backup/Pipeline/Task.py

@@ -39,7 +39,9 @@ def compression(self, method=None):
         return parse_method(self.compression_method)
 
     def is_compressed(self):
-        if self.compression() != 'none':
+        if self.compression() == 'auto' and hasattr(self, "can_compress"):
+            return self.can_compress()
+        elif self.compression() != 'none':
             return True
         return False