Moving 'compression' and 'threads' up to tar-only

Author: timvaillancourt

.gitignore

@@ -1,5 +1,6 @@
 bin
 build
 rpmbuild
+pex
 *.pyc
 .idea

Makefile

@@ -34,4 +34,4 @@ rpm: clean
 	rpmbuild -D "_topdir $(PWD)/rpmbuild" -D "version $(VERSION)" -bb rpmbuild/SPECS/$(NAME).spec
 
 clean:
-	rm -rf bin build rpmbuild $(NAME).egg-info 2>/dev/null
+	rm -rf bin build rpmbuild $(NAME).egg-info pex 2>/dev/null

mongodb_consistent_backup/Archive/Archive.py

@@ -13,15 +13,16 @@ def __init__(self, config, backup_dir):
 
     def init(self):
         archive_method = self.config.archive.method
-        if not archive_method or archive_method is "none":
+        if not archive_method or archive_method.lower() is "none":
             logging.info("Archiving disabled, skipping")
         else:
             config_vars = ""
+            method = archive_method.lower()
             for key in self.config.archive:
                 config_vars += "%s=%s," % (key, self.config.archive[key])
-            logging.info("Using archiving method: %s (options: %s)" % (archive_method, str(config_vars[:-1])))
+            logging.info("Using archiving method: %s (options: %s)" % (method, str(config_vars[:-1])))
             try:
-                self._archiver = globals()[archive_method.capitalize()](
+                self._archiver = globals()[method.capitalize()](
                     self.config,
                     self.backup_dir
                 )

mongodb_consistent_backup/Archive/Tar/Tar.py

@@ -22,8 +22,8 @@ class Tar:
     def __init__(self, config, backup_base_dir):
         self.config          = config
         self.backup_base_dir = backup_base_dir
-        self.compression     = self.config.archive.compression
-        self.thread_count    = self.config.archive.threads
+        self.compression     = self.config.archive.tar.compression
+        self.thread_count    = self.config.archive.tar.threads
         self.verbose         = self.config.verbose
         self.binary          = "tar"
 

mongodb_consistent_backup/Archive/Tar/__init__.py

@@ -2,6 +2,8 @@
 
 
 def config(parser):
-    parser.add_argument("--archive.compression", dest="archive.compression",
+    parser.add_argument("--archive.tar.compression", dest="archive.tar.compression",
                         help="Archiver compression method (default: gzip)", default='gzip', choices=['gzip', 'none'])
+    parser.add_argument("--archive.tar.threads", dest="archive.tar.threads", 
+		        help="Number of threads to use in archive phase (default: 1-per-CPU)", default=0, type=int)
     return parser

mongodb_consistent_backup/Archive/__init__.py

@@ -3,5 +3,4 @@
 
 def config(parser):
     parser.add_argument("--archive.method", dest="archive.method", help="Archiver method (default: tar)", default='tar', choices=['tar','none'])
-    parser.add_argument("--archive.threads", dest="archive.threads", help="Number of threads to use in archive phase (default: 1-per-CPU)", default=0, type=int)
     return parser

mongodb_consistent_backup/Backup/Backup.py

@@ -15,18 +15,19 @@ def __init__(self, config, backup_dir, secondaries, config_server=None):
 
     def init(self):
         backup_method = self.config.backup.method
-        if not backup_method or backup_method is "none":
+        if not backup_method or backup_method.lower() is "none":
             raise Exception, 'Must specify a backup method!', None
-        logging.info("Using backup method: %s" % backup_method)
+        method = backup_method.lower()
+        logging.info("Using backup method: %s" % method)
         try:
-            self._method = globals()[backup_method.capitalize()](
+            self._method = globals()[method.capitalize()](
                 self.config,
                 self.backup_dir,
                 self.secondaries,
                 self.config_server
             )
         except Exception, e:
-            raise Exception, "Problem performing %s! Error: %s" % (backup_method, e), None
+            raise Exception, "Problem performing %s! Error: %s" % (method, e), None
 
     def is_compressed(self):
         if self._method:

mongodb_consistent_backup/Notify/Notify.py

@@ -2,18 +2,20 @@
 
 class Notify:
     def __init__(self, config):
-        self.config = config
+        self.config    = config
+
         self._notifier = None
         self.init()
 
     def init(self):
         notify_method = self.config.notify.method
-        if not notify_method or notify_method is "none":
+        if not notify_method or notify_method.lower() is "none":
             logging.info("Notifying disabled, skipping")
         else:
-            logging.info("Using notify method: %s" % notify_method)
+            method = notify_method.lower()
+            logging.info("Using notify method: %s" % method)
             try:
-                self._notifier = globals()[notify_method.capitalize()](self.config)
+                self._notifier = globals()[method.capitalize()](self.config)
             except Exception, e:
                 raise e
 

mongodb_consistent_backup/Upload/Upload.py

@@ -14,21 +14,22 @@ def __init__(self, config, base_dir, backup_dir):
 
     def init(self):
         upload_method = self.config.upload.method
-        if not upload_method or upload_method is "none":
+        if not upload_method or upload_method.lower() is "none":
             logging.info("Uploading disabled, skipping")
         else:
             #TODO Remove this line and move to  S3 Lib for checking
             # if self.config.upload.method == "s3" and self.config.upload.s3.bucket_name and self.config.upload.s3.bucket_prefix and self.config.upload.s3.access_key and self.config.upload.s3.secret_key:
 
-            logging.info("Using upload method: %s" % upload_method)
+            method = upload_method.lower()
+            logging.info("Using upload method: %s" % method)
             try:
-                self._uploader = globals()[upload_method.capitalize()](
+                self._uploader = globals()[method.capitalize()](
                     self.config,
                     self.base_dir,
                     self.backup_dir
                 )
             except Exception, e:
-                raise Exception, "Problem settings up %s Uploader Error: %s" % (self.config.upload.method, e), None
+                raise Exception, "Problem settings up %s Uploader Error: %s" % (method, e), None
 
     def upload(self):
         if self._uploader:

scripts/build.sh

@@ -8,6 +8,8 @@ rootdir=$(readlink -f $(dirname $0)/..)
 srcdir=${rootdir}/${mod_name}
 bindir=${rootdir}/bin
 builddir=${rootdir}/build
+cachedir=${rootdir}/cache
+pexcachedir=${cachedir}/cache/pex
 venvdir=${builddir}/venv
 output_file=${bindir}/${name}
 require_file=${builddir}/requirements.txt
@@ -85,15 +87,16 @@ if [ -d ${srcdir} ]; then
 		exit 1
 	fi
 	source ${venvdir}/bin/activate
-		
+
 	${venvdir}/bin/python2.7 ${venvdir}/bin/pip install pex requests
 	if [ $? -gt 0 ]; then
 		echo "Failed to install pex utility for building!"
 		exit 1
 	fi
 
 	[ ! -d ${bindir} ] && mkdir -p ${bindir}
-	${venvdir}/bin/python2.7 ${venvdir}/bin/pex --disable-cache -o ${output_file} -m ${mod_name} -r ${require_file} ${builddir}
+	[ ! -d ${pexcachedir} ] && mkdir -p ${pexcachedir}
+	${venvdir}/bin/python2.7 ${venvdir}/bin/pex --cache-root ${pexcachedir}  -o ${output_file} -m ${mod_name} -r ${require_file} ${builddir}
 	if [ $? -lt 1 ] && [ -x ${output_file} ]; then
 		echo "pex executable written to '$output_file'"
 	else