Merge pull request #98 from timvaillancourt/MCB_1.0-bugfix-v9

timvaillancourt · web-flow · commit 3c1b9234c07a · 2017-04-10T23:13:49.000+02:00
Mcb 1.0 bugfix v9
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,5 @@
 bin
 build
-rpmbuild
 tmp
 *.pyc
 .idea
diff --git a/Makefile b/Makefile
@@ -18,23 +18,24 @@ install: bin/mongodb-consistent-backup
 	rm -rf bin build 2>/dev/null
 	mkdir -p $(BINDIR) $(SHAREDIR)/$(NAME) || true
 	install -m 0755 bin/mongodb-consistent-backup $(BINDIR)/mongodb-consistent-backup
-	install -m 0644 conf/example.yml $(SHAREDIR)/$(NAME)/example.yml
+	install -m 0644 conf/mongodb-consistent-backup.example.yml $(SHAREDIR)/$(NAME)/example.yml
 	install -m 0644 LICENSE $(SHAREDIR)/$(NAME)/LICENSE
 	install -m 0644 README.rst $(SHAREDIR)/$(NAME)/README.rst
 
 uninstall:
 	rm -f $(BINDIR)/mongodb-consistent-backup
 	rm -rf $(SHAREDIR)/$(NAME)
 
-rpm: clean
-	mkdir -p rpmbuild/{SPECS,SOURCES/$(NAME)}
-	cp -dpR $(NAME) conf Makefile setup.py scripts requirements.txt LICENSE README.rst VERSION rpmbuild/SOURCES/$(NAME)
-	install scripts/$(NAME).spec rpmbuild/SPECS/$(NAME).spec
-	tar --remove-files -C rpmbuild/SOURCES -czf rpmbuild/SOURCES/$(NAME).tar.gz $(NAME)
-	rpmbuild -D "_topdir $(PWD)/rpmbuild" -D "version $(VERSION)" -bb rpmbuild/SPECS/$(NAME).spec
+rpm: bin/mongodb-consistent-backup
+	rm -rf build/rpm 2>/dev/null || true
+	mkdir -p build/rpm/SOURCES
+	cp -f $(PWD)/{LICENSE,README.rst} build/rpm/SOURCES
+	cp -f $(PWD)/bin/mongodb-consistent-backup build/rpm/SOURCES/mongodb-consistent-backup
+	cp -f $(PWD)/conf/mongodb-consistent-backup.example.yml build/rpm/SOURCES/mongodb-consistent-backup.yml
+	rpmbuild -D "_topdir $(PWD)/build/rpm" -D "version $(VERSION)" -bb scripts/$(NAME).spec
 
 docker:
 	docker build --no-cache -t mongodb_consistent_backup .
 
 clean:
-	rm -rf bin build rpmbuild $(NAME).egg-info tmp 2>/dev/null
+	rm -rf bin build $(NAME).egg-info tmp 2>/dev/null
diff --git a/conf/mongodb-consistent-backup.example.yml b/conf/mongodb-consistent-backup.example.yml
@@ -4,9 +4,10 @@ production:
   #username: 
   #password:
   #authdb: admin
+  log_dir: /tmp
   backup:
     method: mongodump
-    name: test
+    name: default
     location: /opt/mongodb/backup
     mongodump:
       binary: /usr/bin/mongodump
diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py
@@ -20,7 +20,7 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, shard
         super(Mongodump, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir)
         self.compression_method = self.config.backup.mongodump.compression
         self.binary             = self.config.backup.mongodump.binary
-        self.user               = self.config.user
+        self.user               = self.config.username
         self.password           = self.config.password
         self.authdb             = self.config.authdb
         self.replsets           = replsets
@@ -152,7 +152,7 @@ def run(self):
                     self.verbose
                 )]
                 self.dump_threads[0].start()
-                self.wait()
+                self.dump_threads[0].join()
 
         self.completed = True
         return self._summary
diff --git a/mongodb_consistent_backup/Common/Config.py b/mongodb_consistent_backup/Common/Config.py
@@ -43,7 +43,8 @@ def makeParser(self):
         parser.add_argument("-n", "--backup.name", dest="backup.name", help="Name of the backup set (required)", type=str)
         parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
         parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
-        parser.add_argument("--lockfile", dest="lockfile", help="Location of lock file (default: /tmp/mongodb_consistent_backup.lock)", default='/tmp/mongodb_consistent_backup.lock', type=str)
+        parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
+        parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb_consistent_backup.lock)", default='/tmp/mongodb_consistent_backup.lock', type=str)
         parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)
         parser.add_argument("--sharding.balancer.ping_secs", dest="sharding.balancer.ping_secs", help="Interval to check balancer state, in seconds (default: 3)", default=3, type=int)
         return parser
diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py
@@ -10,7 +10,7 @@
 class DB:
     def __init__(self, uri, config, do_replset=False, read_pref='primaryPreferred', do_connect=True, conn_timeout=5000, retries=5):
         self.uri          = uri
-        self.username     = config.user
+        self.username     = config.username
         self.password     = config.password
         self.authdb       = config.authdb
         self.do_replset   = do_replset
diff --git a/mongodb_consistent_backup/Logger.py b/mongodb_consistent_backup/Logger.py
@@ -0,0 +1,69 @@
+import logging
+import os
+
+from gzip import GzipFile
+
+
+class Logger:
+    def __init__(self, config, backup_time):
+        self.config      = config
+        self.backup_time = backup_time
+
+        self.log_level = logging.INFO
+        if self.config.verbose:
+            self.log_level = logging.DEBUG
+
+        self.do_file_log = False
+        if self.config.log_dir is not '':
+            if os.path.isdir(self.config.log_dir):
+                self.do_file_log = True
+            else:
+                print("ERROR: Log directory: %s does not exist! Skipping file-based logging" % self.config.log_dir)
+
+        self.log_format = '[%(asctime)s] [%(levelname)s] [%(processName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s'
+        self.file_log   = None
+        self.last_log   = None
+
+    def start(self):
+        try:
+            logging.basicConfig(level=self.log_level, format=self.log_format)
+            if self.do_file_log:
+                self.current_log_file = os.path.join(self.config.log_dir, "backup.log")
+                self.backup_log_file  = os.path.join(self.config.log_dir, "backup.%s.log" % self.backup_time)
+                self.file_log = logging.FileHandler(self.backup_log_file)
+                self.file_log.setLevel(self.log_level)
+                self.file_log.setFormatter(logging.Formatter(self.log_format))
+                logging.getLogger('').addHandler(self.file_log)
+        except OSError, e:
+            logging.warning("Could not start file log handler, writing to stdout only")
+            pass
+
+    def close(self):
+        if self.file_log:
+            self.file_log.close()
+
+    def compress_last(self):
+        gz_log = None
+        try:
+            if not os.path.isfile(self.last_log):
+                return
+            gz_file = "%s.gz" % self.last_log
+            gz_log  = GzipFile(gz_file, "w+")
+            with open(self.last_log) as f:
+                for line in f:
+                    gz_log.write(line)
+            os.remove(self.last_log)
+        finally:
+            if gz_log:
+                gz_log.close()
+
+    def rotate(self):
+        if self.do_file_log:
+            if os.path.islink(self.current_log_file):
+                 logging.info("Rotating previous log file")
+                 self.last_log = os.readlink(self.current_log_file)
+                 if self.last_log == self.backup_log_file:
+                     return
+                 os.remove(self.current_log_file)
+                 self.compress_last()
+            os.symlink(self.backup_log_file, self.current_log_file)
diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py
@@ -10,6 +10,7 @@
 from Backup import Backup
 from Common import Config, DB, Lock, MongoUri, Timer
 from Errors import Error, NotifyError, OperationError
+from Logger import Logger
 from Notify import Notify
 from Oplog import Tailer, Resolver
 from Replication import Replset, ReplsetSharded
@@ -44,6 +45,10 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
         self.manager                  = Manager()
         self.timer                    = Timer(self.manager)
         self.timer_name               = "mongodb_consistent_backup.%s" % self.__class__.__name__
+        self.backup_time              = datetime.now().strftime("%Y%m%d_%H%M")
+        self.logger                   = None
+        self.current_log_file         = None
+        self.backup_log_file          = None
 
         try:
             self.setup_config()
@@ -65,11 +70,11 @@ def setup_config(self):
             sys.exit(1)
 
     def setup_logger(self):
-        self.log_level = logging.INFO
-        if self.config.verbose:
-            self.log_level = logging.DEBUG
-        logging.basicConfig(level=self.log_level,
-                            format='[%(asctime)s] [%(levelname)s] [%(processName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s')
+        try:
+            self.logger = Logger(self.config, self.backup_time)
+            self.logger.start()
+        except Exception, e:
+            self.exception("Could not start logger: %s" % e, e)
 
     def setup_signal_handlers(self):
         try:
@@ -80,7 +85,6 @@ def setup_signal_handlers(self):
             sys.exit(1)
 
     def set_backup_dirs(self):
-        self.backup_time = datetime.now().strftime("%Y%m%d_%H%M")
         self.backup_root_directory    = os.path.join(self.config.backup.location, self.config.backup.name)
         self.backup_latest_symlink    = os.path.join(self.backup_root_directory, "latest")
         self.backup_previous_symlink  = os.path.join(self.backup_root_directory, "previous")
@@ -107,11 +111,11 @@ def get_db_conn(self):
     def get_lock(self):
         # noinspection PyBroadException
         try:
-            if not self.config.lockfile:
-                self.config.lockfile = '/tmp/%s.lock' % self.program_name
-            self.lock = Lock(self.config.lockfile)
+            if not self.config.lock_file:
+                self.config.lock_file = '/tmp/%s.lock' % self.program_name
+            self.lock = Lock(self.config.lock_file)
         except Exception:
-            logging.fatal("Could not acquire lock: '%s'! Is another %s process running? Exiting" % (self.config.lockfile, self.program_name))
+            logging.fatal("Could not acquire lock: '%s'! Is another %s process running? Exiting" % (self.config.lock_file, self.program_name))
             self.cleanup_and_exit(None, None)
 
     def release_lock(self):
@@ -180,6 +184,11 @@ def cleanup_and_exit(self, code, frame):
             self.db.close()
 
         logging.info("Cleanup complete, exiting")
+
+        if self.logger:
+            self.logger.rotate()
+            self.logger.close()
+
         self.release_lock()
         sys.exit(1)
 
@@ -448,4 +457,6 @@ def run(self):
         self.update_symlinks()
         logging.info("Completed %s in %.2f sec" % (self.program_name, self.timer.duration(self.timer_name)))
 
+        self.logger.rotate()
+        self.logger.close()
         self.release_lock()
diff --git a/mongodb_consistent_backup/Oplog/Tailer/Tailer.py b/mongodb_consistent_backup/Oplog/Tailer/Tailer.py
@@ -17,7 +17,7 @@ class Tailer(Task):
     def __init__(self, manager, config, timer, base_dir, backup_dir, replsets, status_secs=15):
         super(Tailer, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir)
         self.backup_name = self.config.name
-        self.user        = self.config.user
+        self.user        = self.config.username
         self.password    = self.config.password
         self.authdb      = self.config.authdb
         self.replsets    = replsets
@@ -30,9 +30,9 @@ def summary(self):
         return self._summary
 
     def prepare_oplog_files(self, shard_name):
-        oplog_dir = os.path.join(self.base_dir, shard_name)
+        oplog_dir = os.path.join(self.backup_dir, shard_name)
         if not os.path.isdir(oplog_dir):
-            os.makedirs(oplog_dir)
+            os.mkdir(oplog_dir)
         oplog_file = os.path.join(oplog_dir, "oplog-tailed.bson")
         return oplog_file
 
diff --git a/mongodb_consistent_backup/Replication/Replset.py b/mongodb_consistent_backup/Replication/Replset.py
@@ -11,9 +11,6 @@ class Replset:
     def __init__(self, config, db):
         self.config       = config
         self.db           = db
-        self.user         = self.config.user
-        self.password     = self.config.password
-        self.authdb       = self.config.authdb
         self.max_lag_secs = self.config.replication.max_lag_secs
         self.min_priority = self.config.replication.min_priority
         self.max_priority = self.config.replication.max_priority
diff --git a/scripts/mongodb_consistent_backup.spec b/scripts/mongodb_consistent_backup.spec
@@ -1,5 +1,9 @@
 %define debug_package	%{nil}
 %define	bin_name	mongodb-consistent-backup
+%define log_dir		/var/log/mongodb-consistent-backup
+%define data_dir	/var/lib/mongodb-consistent-backup
+%define run_user	mongodb_consistent_backup
+%define run_group	mongodb_consistent_backup
 
 Name:		mongodb_consistent_backup
 Version:	%{version}
@@ -9,7 +13,10 @@ Summary:	MongoDB Consistent Backup Tool
 Group:		Software/Database
 License:	Apache License Version 2.0
 URL:		https://github.com/Percona-Lab/mongodb_consistent_backup
-Source:		%{name}.tar.gz
+Source0:	%{bin_name}
+Source1:	%{bin_name}.yml
+Source2:	LICENSE
+Source3:	README.rst
 Prefix:		/usr
 
 # Use CentOS SCL python27 (https://www.softwarecollections.org/en/scls/rhscl/python27/) on CentOS 6 (RHEL6 untested)
@@ -27,23 +34,44 @@ Prefix:		/usr
 Tool for getting consistent backups from MongoDB Clusters and ReplicaSet
 
 
-%prep
-%setup -q -n %{name}
+%install
+mkdir -p %{buildroot}%{_sysconfdir}/cron.d %{buildroot}%{prefix}/bin %{buildroot}/usr/share/%{name}
 
+install -m 0755 %{SOURCE0} %{buildroot}%{prefix}/bin/%{bin_name}
+install -m 0644 %{SOURCE1} %{buildroot}/usr/share/%{name}/%{bin_name}.example.yml
+install -m 0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/%{bin_name}.yml
+install -m 0644 %{SOURCE2} %{buildroot}/usr/share/%{name}/LICENSE
+install -m 0644 %{SOURCE3} %{buildroot}/usr/share/%{name}/README.rst
 
-%build
-make
 
+# Generate cron.d file:
+%{__cat} <<EOF >%{buildroot}%{_sysconfdir}/cron.d/%{name}
+### Uncomment and adjust time to enable backups (default time below is 00:00 every day):
+#
+#0 0 * * *	%{run_user}	/usr/bin/mongodb-consistent-backup --config=/etc/mongodb-consistent-backup.yml >/dev/null 2>&1
+EOF
+
+# Change /etc config file to use rpm paths for logs and data
+sed -i \
+	-e s@log_dir:\ /tmp@log_dir:\ %{log_dir}@g \
+	-e s@location:\ /opt/mongodb/backup@location:\ %{data_dir}@g \
+	%{buildroot}%{_sysconfdir}/%{bin_name}.yml
+
+
+%pre
+/usr/bin/getent group %{run_group} >/dev/null 2>&1 || /usr/sbin/groupadd -r %{run_group}
+/usr/bin/getent passwd %{run_user} >/dev/null 2>&1 || /usr/sbin/useradd -r -d %{data_dir} -g %{run_group} -s /sbin/nologin %{run_user}
 
-%install
-mkdir -p %{buildroot}%{_sysconfdir}
-install -m 0644 conf/mongodb-consistent-backup.example.yml %{buildroot}%{_sysconfdir}/%{bin_name}.yml
 
-make PREFIX=%{prefix} DESTDIR=%{buildroot} install
+%post
+[ ! -d %{data_dir} ] && mkdir -m 0750 -p %{data_dir}
+[ ! -d %{log_dir} ] && mkdir -m 0755 -p %{log_dir}
+chown %{run_user}:%{run_group} %{data_dir} %{log_dir}
 
 
 %files
-%{_sysconfdir}/%{bin_name}.yml
+%config(noreplace) %{_sysconfdir}/%{bin_name}.yml
+%config(noreplace) %{_sysconfdir}/cron.d/%{name}
 %{prefix}/bin/%{bin_name}
 %{prefix}/share/%{name}/%{bin_name}.example.yml
 %{prefix}/share/%{name}/LICENSE

-Original file line number
+Diff line change
@@ @@ -1,6 +1,5 @@ @@
 bin
 build
 -rpmbuild
 tmp
 *.pyc
 .idea