Got rpmbuilds working again, made it much simpler. Add log support

Author: timvaillancourt

.gitignore

@@ -1,6 +1,5 @@
 bin
 build
-rpmbuild
 tmp
 *.pyc
 .idea

Makefile

@@ -18,23 +18,24 @@ install: bin/mongodb-consistent-backup
 	rm -rf bin build 2>/dev/null
 	mkdir -p $(BINDIR) $(SHAREDIR)/$(NAME) || true
 	install -m 0755 bin/mongodb-consistent-backup $(BINDIR)/mongodb-consistent-backup
-	install -m 0644 conf/example.yml $(SHAREDIR)/$(NAME)/example.yml
+	install -m 0644 conf/mongodb-consistent-backup.example.yml $(SHAREDIR)/$(NAME)/example.yml
 	install -m 0644 LICENSE $(SHAREDIR)/$(NAME)/LICENSE
 	install -m 0644 README.rst $(SHAREDIR)/$(NAME)/README.rst
 
 uninstall:
 	rm -f $(BINDIR)/mongodb-consistent-backup
 	rm -rf $(SHAREDIR)/$(NAME)
 
-rpm: clean
-	mkdir -p rpmbuild/{SPECS,SOURCES/$(NAME)}
-	cp -dpR $(NAME) conf Makefile setup.py scripts requirements.txt LICENSE README.rst VERSION rpmbuild/SOURCES/$(NAME)
-	install scripts/$(NAME).spec rpmbuild/SPECS/$(NAME).spec
-	tar --remove-files -C rpmbuild/SOURCES -czf rpmbuild/SOURCES/$(NAME).tar.gz $(NAME)
-	rpmbuild -D "_topdir $(PWD)/rpmbuild" -D "version $(VERSION)" -bb rpmbuild/SPECS/$(NAME).spec
+rpm: bin/mongodb-consistent-backup
+	rm -rf build/rpm 2>/dev/null || true
+	mkdir -p build/rpm/SOURCES
+	cp -f $(PWD)/{LICENSE,README.rst} build/rpm/SOURCES
+	cp -f $(PWD)/bin/mongodb-consistent-backup build/rpm/SOURCES/mongodb-consistent-backup
+	cp -f $(PWD)/conf/mongodb-consistent-backup.example.yml build/rpm/SOURCES/mongodb-consistent-backup.yml
+	rpmbuild -D "_topdir $(PWD)/build/rpm" -D "version $(VERSION)" -bb scripts/$(NAME).spec
 
 docker:
 	docker build --no-cache -t mongodb_consistent_backup .
 
 clean:
-	rm -rf bin build rpmbuild $(NAME).egg-info tmp 2>/dev/null
+	rm -rf bin build $(NAME).egg-info tmp 2>/dev/null

conf/mongodb-consistent-backup.example.yml

@@ -4,9 +4,10 @@ production:
   #username: 
   #password:
   #authdb: admin
+  log: /tmp/mongodb_consistent_backup.log
   backup:
     method: mongodump
-    name: test
+    name: default
     location: /opt/mongodb/backup
     mongodump:
       binary: /usr/bin/mongodump

mongodb_consistent_backup/Common/Config.py

@@ -43,7 +43,8 @@ def makeParser(self):
         parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
         parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
         parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
-        parser.add_argument("--lockfile", dest="lockfile", help="Location of lock file (default: /tmp/mongodb_consistent_backup.lock)", default='/tmp/mongodb_consistent_backup.lock', type=str)
+        parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
+        parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb_consistent_backup.lock)", default='/tmp/mongodb_consistent_backup.lock', type=str)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)
         parser.add_argument("--sharding.balancer.ping_secs", dest="sharding.balancer.ping_secs", help="Interval to check balancer state, in seconds (default: 3)", default=3, type=int)
         return parser

mongodb_consistent_backup/Logger.py

@@ -0,0 +1,69 @@
+import logging
+import os
+
+from gzip import GzipFile
+
+
+class Logger:
+    def __init__(self, config, backup_time):
+        self.config      = config
+        self.backup_time = backup_time
+
+        self.log_level = logging.INFO
+        if self.config.verbose:
+            self.log_level = logging.DEBUG
+
+        self.do_file_log = False
+        if self.config.log_dir is not '':
+            if os.path.isdir(self.config.log_dir):
+                self.do_file_log = True
+            else:
+                print("ERROR: Log directory: %s does not exist! Skipping file-based logging" % self.config.log_dir)
+
+        self.log_format = '[%(asctime)s] [%(levelname)s] [%(processName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s'
+        self.file_log   = None
+        self.last_log   = None
+
+    def start(self):
+        try:
+            logging.basicConfig(level=self.log_level, format=self.log_format)
+            if self.do_file_log:
+                self.current_log_file = os.path.join(self.config.log_dir, "backup.log")
+                self.backup_log_file  = os.path.join(self.config.log_dir, "backup.%s.log" % self.backup_time)
+                self.file_log = logging.FileHandler(self.backup_log_file)
+                self.file_log.setLevel(self.log_level)
+                self.file_log.setFormatter(logging.Formatter(self.log_format))
+                logging.getLogger('').addHandler(self.file_log)
+        except OSError, e:
+            logging.warning("Could not start file log handler, writing to stdout only")
+            pass
+
+    def close(self):
+        if self.file_log:
+            self.file_log.close()
+
+    def compress_last(self):
+        gz_log = None
+        try:
+            if not os.path.isfile(self.last_log):
+                return
+            gz_file = "%s.gz" % self.last_log
+            gz_log  = GzipFile(gz_file, "w+")
+            with open(self.last_log) as f:
+                for line in f:
+                    gz_log.write(line)
+            os.remove(self.last_log)
+        finally:
+            if gz_log:
+                gz_log.close()
+
+    def rotate(self):
+        if self.do_file_log:
+            if os.path.islink(self.current_log_file):
+                 logging.info("Rotating previous log file")
+                 self.last_log = os.readlink(self.current_log_file)
+                 if self.last_log == self.backup_log_file:
+                     return
+                 os.remove(self.current_log_file)
+                 self.compress_last()
+            os.symlink(self.backup_log_file, self.current_log_file)

mongodb_consistent_backup/Main.py

@@ -10,6 +10,7 @@
 from Backup import Backup
 from Common import Config, DB, Lock, MongoUri, Timer
 from Errors import Error, NotifyError, OperationError
+from Logger import Logger
 from Notify import Notify
 from Oplog import Tailer, Resolver
 from Replication import Replset, ReplsetSharded
@@ -44,6 +45,10 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
         self.manager                  = Manager()
         self.timer                    = Timer(self.manager)
         self.timer_name               = "mongodb_consistent_backup.%s" % self.__class__.__name__
+        self.backup_time              = datetime.now().strftime("%Y%m%d_%H%M")
+        self.logger                   = None
+        self.current_log_file         = None
+        self.backup_log_file          = None
 
         try:
             self.setup_config()
@@ -65,11 +70,11 @@ def setup_config(self):
             sys.exit(1)
 
     def setup_logger(self):
-        self.log_level = logging.INFO
-        if self.config.verbose:
-            self.log_level = logging.DEBUG
-        logging.basicConfig(level=self.log_level,
-                            format='[%(asctime)s] [%(levelname)s] [%(processName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s')
+        try:
+            self.logger = Logger(self.config, self.backup_time)
+            self.logger.start()
+        except Exception, e:
+            self.exception("Could not start logger: %s" % e, e)
 
     def setup_signal_handlers(self):
         try:
@@ -80,7 +85,6 @@ def setup_signal_handlers(self):
             sys.exit(1)
 
     def set_backup_dirs(self):
-        self.backup_time = datetime.now().strftime("%Y%m%d_%H%M")
         self.backup_root_directory    = os.path.join(self.config.backup.location, self.config.backup.name)
         self.backup_latest_symlink    = os.path.join(self.backup_root_directory, "latest")
         self.backup_previous_symlink  = os.path.join(self.backup_root_directory, "previous")
@@ -107,11 +111,11 @@ def get_db_conn(self):
     def get_lock(self):
         # noinspection PyBroadException
         try:
-            if not self.config.lockfile:
-                self.config.lockfile = '/tmp/%s.lock' % self.program_name
-            self.lock = Lock(self.config.lockfile)
+            if not self.config.lock_file:
+                self.config.lock_file = '/tmp/%s.lock' % self.program_name
+            self.lock = Lock(self.config.lock_file)
         except Exception:
-            logging.fatal("Could not acquire lock: '%s'! Is another %s process running? Exiting" % (self.config.lockfile, self.program_name))
+            logging.fatal("Could not acquire lock: '%s'! Is another %s process running? Exiting" % (self.config.lock_file, self.program_name))
             self.cleanup_and_exit(None, None)
 
     def release_lock(self):
@@ -180,6 +184,11 @@ def cleanup_and_exit(self, code, frame):
             self.db.close()
 
         logging.info("Cleanup complete, exiting")
+
+        if self.logger:
+            self.logger.rotate()
+            self.logger.close()
+
         self.release_lock()
         sys.exit(1)
 
@@ -448,4 +457,6 @@ def run(self):
         self.update_symlinks()
         logging.info("Completed %s in %.2f sec" % (self.program_name, self.timer.duration(self.timer_name)))
 
+        self.logger.rotate()
+        self.logger.close()
         self.release_lock()

scripts/mongodb_consistent_backup.spec

@@ -1,5 +1,9 @@
 %define debug_package	%{nil}
 %define	bin_name	mongodb-consistent-backup
+%define log_dir		/var/log/mongodb-consistent-backup
+%define data_dir	/var/lib/mongodb-consistent-backup
+%define run_user	mongodb_consistent_backup
+%define run_group	mongodb_consistent_backup
 
 Name:		mongodb_consistent_backup
 Version:	%{version}
@@ -9,7 +13,10 @@ Summary:	MongoDB Consistent Backup Tool
 Group:		Software/Database
 License:	Apache License Version 2.0
 URL:		https://github.com/Percona-Lab/mongodb_consistent_backup
-Source:		%{name}.tar.gz
+Source0:	%{bin_name}
+Source1:	%{bin_name}.yml
+Source2:	LICENSE
+Source3:	README.rst
 Prefix:		/usr
 
 # Use CentOS SCL python27 (https://www.softwarecollections.org/en/scls/rhscl/python27/) on CentOS 6 (RHEL6 untested)
@@ -27,23 +34,44 @@ Prefix:		/usr
 Tool for getting consistent backups from MongoDB Clusters and ReplicaSet
 
 
-%prep
-%setup -q -n %{name}
+%install
+mkdir -p %{buildroot}%{_sysconfdir}/cron.d %{buildroot}%{prefix}/bin %{buildroot}/usr/share/%{name}
 
+install -m 0755 %{SOURCE0} %{buildroot}%{prefix}/bin/%{bin_name}
+install -m 0644 %{SOURCE1} %{buildroot}/usr/share/%{name}/%{bin_name}.example.yml
+install -m 0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/%{bin_name}.yml
+install -m 0644 %{SOURCE2} %{buildroot}/usr/share/%{name}/LICENSE
+install -m 0644 %{SOURCE3} %{buildroot}/usr/share/%{name}/README.rst
 
-%build
-make
 
+# Generate cron.d file:
+%{__cat} <<EOF >%{buildroot}%{_sysconfdir}/cron.d/%{name}
+### Uncomment and adjust time to enable backups (default time below is 00:00 every day):
+#
+#0 0 * * *	%{run_user}	/usr/bin/mongodb-consistent-backup --config=/etc/mongodb-consistent-backup.yml >/dev/null 2>&1
+EOF
+
+# Change /etc config file to use rpm paths for logs and data
+sed -i \
+	-e s@/tmp/mongodb_consistent_backup.log@%{log_dir}/mongodb-consistent-backup.log@g \
+	-e s@/opt/mongodb/backup@%{data_dir}@g \
+	%{buildroot}%{_sysconfdir}/%{bin_name}.yml
+
+
+%pre
+/usr/bin/getent group %{run_group} >/dev/null 2>&1 || /usr/sbin/groupadd -r %{run_group}
+/usr/bin/getent passwd %{run_user} >/dev/null 2>&1 || /usr/sbin/useradd -r -d %{data_dir} -g %{run_group} -s /sbin/nologin %{run_user}
 
-%install
-mkdir -p %{buildroot}%{_sysconfdir}
-install -m 0644 conf/mongodb-consistent-backup.example.yml %{buildroot}%{_sysconfdir}/%{bin_name}.yml
 
-make PREFIX=%{prefix} DESTDIR=%{buildroot} install
+%post
+[ ! -d %{data_dir} ] && mkdir -m 0750 -p %{data_dir}
+[ ! -d %{log_dir} ] && mkdir -m 0755 -p %{log_dir}
+chown %{run_user}:%{run_group} %{data_dir} %{log_dir}
 
 
 %files
 %{_sysconfdir}/%{bin_name}.yml
+%{_sysconfdir}/cron.d/%{name}
 %{prefix}/bin/%{bin_name}
 %{prefix}/share/%{name}/%{bin_name}.example.yml
 %{prefix}/share/%{name}/LICENSE