Merge pull request #132 from timvaillancourt/mongodump_compression_fixv1

1.0.2: Default backup compression to 'auto' so 'none' can be set explicitly

Author: timvaillancourt

conf/mongodb-consistent-backup.example.conf

@@ -10,9 +10,9 @@ production:
     name: default
     location: /var/lib/mongodb-consistent-backup
   #  mongodump:
-  #    binary: [path]           (default: /usr/bin/mongodump)
-  #    compression: [none|gzip] (default: true - if supported)
-  #    threads: [1-16]          (default: auto-generated - shards/cpu)
+  #    binary: [path]                (default: /usr/bin/mongodump)
+  #    compression: [auto|none|gzip] (default: auto - enable gzip if supported)
+  #    threads: [1-16]               (default: auto-generated - shards/cpu)
   #replication:
   #  max_lag_secs: [1+]        (default: 5)
   #  min_priority: [0-999]     (default: 0)
@@ -23,14 +23,14 @@ production:
   #    wait_secs: [1+] (default: 300)
   #    ping_secs: [1+] (default: 3)
   #oplog:
-  #  compression: [none|gzip] (default: true - if used by backup stage)
+  #  compression: [none|gzip] (default: gzip - if gzip is used by backup stage)
   #  resolver_threads: [1+]   (default: 2 per CPU)
   #  tailer:
   #    status_interval: 30
   archive:
     method: tar
   #  tar:
-  #    compression: [none|gzip] (default: gzip, none if backup is compressed)
+  #    compression: [none|gzip] (default: gzip - none if backup is already compressed)
   #    threads: [1+]            (default: 1 per CPU)
   #  zbackup:
   #    binary: [path]        (default: /usr/bin/zbackup)

mongodb_consistent_backup/Backup/Mongodump/Mongodump.py

@@ -26,7 +26,7 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         self.replsets           = replsets
         self.sharding           = sharding
 
-        self.compression_supported = ['none', 'gzip']
+        self.compression_supported = ['auto', 'none', 'gzip']
         self.version               = 'unknown'
         self.threads_max           = 16
         self.config_replset        = False
@@ -40,10 +40,15 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         with hide('running', 'warnings'), settings(warn_only=True):
             self.version = local("%s --version|awk 'NR >1 {exit}; /version/{print $NF}'" % self.binary, capture=True)
 
-        if self.can_gzip() and self.compression() == 'none':
-            self.compression('gzip')
-        elif self.compression() == 'gzip':
-            logging.warning("mongodump gzip compression requested on binary that does not support gzip!")
+        self.choose_compression()
+
+    def choose_compression(self):
+        if self.can_gzip():
+            if self.compression() == 'auto':
+                logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression")
+                self.compression('gzip')
+	elif self.compression() == 'gzip':
+            raise OperationError("mongodump gzip compression requested on binary that does not support gzip!")
 
     def can_gzip(self):
         if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK):

mongodb_consistent_backup/Backup/Mongodump/__init__.py

@@ -5,8 +5,8 @@ def config(parser):
     parser.add_argument("--backup.mongodump.binary", dest="backup.mongodump.binary",
                         help="Path to 'mongodump' binary (default: /usr/bin/mongodump)", default='/usr/bin/mongodump')
     parser.add_argument("--backup.mongodump.compression", dest="backup.mongodump.compression",
-                        help="Compression method to use on backup (default: gzip)", default="gzip",
-                        choices=["none", "gzip"])
+                        help="Compression method to use on backup (default: auto)", default="auto",
+                        choices=["auto", "none", "gzip"])
     parser.add_argument("--backup.mongodump.threads", dest="backup.mongodump.threads",
                         help="Number of threads to use for each mongodump process. There is 1 x mongodump per shard, be careful! (default: shards/CPUs)",
                         default=0, type=int)

mongodb_consistent_backup/Backup/__init__.py

@@ -1,3 +1,8 @@
 from Backup import Backup
 
 
+def config(parser):
+    parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
+    parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
+    parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
+    return parser

mongodb_consistent_backup/Common/Config.py

@@ -54,9 +54,6 @@ def makeParser(self):
         parser.add_argument("-u", "--username", dest="username", help="MongoDB Authentication Username (for optional auth)", type=str)
         parser.add_argument("-p", "--password", dest="password", help="MongoDB Authentication Password (for optional auth)", type=str)
         parser.add_argument("-a", "--authdb", dest="authdb", help="MongoDB Auth Database (for optional auth - default: admin)", default='admin', type=str)
-        parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
-        parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
-        parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
         parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
         parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)

scripts/build.sh

@@ -100,7 +100,7 @@ if [ -d ${srcdir} ]; then
 	if [ ! -d ${pexdir} ]; then
 		mkdir -p ${pexdir}
 	else
-		rm -f ${pexdir}/${mod_name}-*.whl 
+		find ${pexdir} -type f -name "${mod_name}-*.whl" -print -delete
 	fi
 	[ ! -d ${bindir} ] && mkdir -p ${bindir}
 	${venvdir}/bin/python2.7 ${venvdir}/bin/pex -o ${output_file} -m ${mod_name} -r ${require_file} --pex-root=${pexdir} ${builddir}