Skip to content

Commit 5df5c14

Browse files
timvaillancourttimvaillancourt
authored
Merge pull request #105 from timvaillancourt/MCB_1.0-bugfix16
MCB_1.0: Bugfixes #16
2 parents 1cc6509 + c63523d commit 5df5c14

File tree

13 files changed

+55
-37
lines changed

13 files changed

+55
-37
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ install: bin/mongodb-consistent-backup
1818
rm -rf bin build 2>/dev/null
1919
mkdir -p $(BINDIR) $(SHAREDIR)/$(NAME) || true
2020
install -m 0755 bin/mongodb-consistent-backup $(BINDIR)/mongodb-consistent-backup
21-
install -m 0644 conf/mongodb-consistent-backup.example.yml $(SHAREDIR)/$(NAME)/example.yml
21+
install -m 0644 conf/mongodb-consistent-backup.example.conf $(SHAREDIR)/$(NAME)/example.conf
2222
install -m 0644 LICENSE $(SHAREDIR)/$(NAME)/LICENSE
2323
install -m 0644 README.rst $(SHAREDIR)/$(NAME)/README.rst
2424

@@ -31,7 +31,7 @@ rpm: bin/mongodb-consistent-backup
3131
mkdir -p build/rpm/SOURCES
3232
cp -f $(PWD)/{LICENSE,README.rst} build/rpm/SOURCES
3333
cp -f $(PWD)/bin/mongodb-consistent-backup build/rpm/SOURCES/mongodb-consistent-backup
34-
cp -f $(PWD)/conf/mongodb-consistent-backup.example.yml build/rpm/SOURCES/mongodb-consistent-backup.yml
34+
cp -f $(PWD)/conf/mongodb-consistent-backup.example.conf build/rpm/SOURCES/mongodb-consistent-backup.conf
3535
rpmbuild -D "_topdir $(PWD)/build/rpm" -D "version $(VERSION)" -bb scripts/$(NAME).spec
3636

3737
docker:

‎README.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ command):
5353

5454
::
5555

56-
yum install python python-devel python-virtualenv gcc make libffi-devel openssl-devel
56+
yum install python python-devel python-virtualenv gcc git make libffi-devel openssl-devel
5757

5858
To install to default '*/usr/local/bin/mongodb-consistent-backup*\ ':
5959

‎conf/mongodb-consistent-backup.example.yml‎ renamed to ‎conf/mongodb-consistent-backup.example.conf‎

File renamed without changes.

‎mongodb_consistent_backup/Archive/Tar/Tar.py‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import logging
33

44
from copy_reg import pickle
5-
from multiprocessing import Pool
5+
from multiprocessing import Pool, TimeoutError
66
from types import MethodType
77

88
from TarThread import TarThread
@@ -27,7 +27,23 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs):
2727
self.compression_method = self.config.archive.tar.compression
2828
self.binary = "tar"
2929

30-
self._pool = None
30+
self._pool = None
31+
self._pooled = []
32+
33+
def wait(self):
34+
if len(self._pooled) > 0:
35+
self._pool.close()
36+
logging.debug("Waiting for tar threads to stop")
37+
while len(self._pooled) > 0:
38+
try:
39+
item = self._pooled[0]
40+
path, result = item
41+
result.get(1)
42+
logging.debug("Archiving completed for directory: %s" % path)
43+
self._pooled.remove(item)
44+
except TimeoutError:
45+
continue
46+
self.stopped = True
3147

3248
def run(self):
3349
try:
@@ -48,18 +64,17 @@ def run(self):
4864
output_file = "%s.tar" % subdir_name
4965
if self.do_gzip():
5066
output_file = "%s.tgz" % subdir_name
51-
self._pool.apply_async(TarThread(subdir_name, output_file, self.do_gzip(), self.verbose, self.binary).run)
67+
result = self._pool.apply_async(TarThread(subdir_name, output_file, self.do_gzip(), self.verbose, self.binary).run)
68+
self._pooled.append((subdir_name, result))
5269
except Exception, e:
5370
self._pool.terminate()
5471
logging.fatal("Could not create tar archiving thread! Error: %s" % e)
5572
raise Error(e)
5673
finally:
57-
self._pool.close()
58-
self._pool.join()
59-
self.stopped = True
74+
self.wait()
6075
self.completed = True
6176

62-
def close(self):
77+
def close(self, code=None, frame=None):
6378
logging.debug("Stopping tar archiving threads")
6479
if not self.stopped and self._pool is not None:
6580
self._pool.terminate()

‎mongodb_consistent_backup/Archive/Tar/TarThread.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@ def run(self):
3434
backup_base_dir = os.path.dirname(self.backup_dir)
3535
backup_base_name = os.path.basename(self.backup_dir)
3636

37-
log_msg = "Archiving and compressing directory: %s" % self.backup_dir
37+
log_msg = "Archiving directory: %s" % self.backup_dir
3838
cmd_flags = ["-C", backup_base_dir, "-cf", self.output_file, "--remove-files", backup_base_name]
3939

4040
if self.do_gzip:
41-
log_msg = "Archiving directory: %s" % self.backup_dir
41+
log_msg = "Archiving and compressing directory: %s" % self.backup_dir
4242
cmd_flags = ["-C", backup_base_dir, "-czf", self.output_file, "--remove-files", backup_base_name]
4343

4444
logging.info(log_msg)

‎mongodb_consistent_backup/Common/Config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ def makeParser(self):
5858
parser.add_argument("-l", "--backup.location", dest="backup.location", help="Base path to store the backup data (required)", type=str)
5959
parser.add_argument("-m", "--backup.method", dest="backup.method", help="Method to be used for backup (default: mongodump)", default='mongodump', choices=['mongodump'])
6060
parser.add_argument("-L", "--log-dir", dest="log_dir", help="Path to write log files to (default: disabled)", default='', type=str)
61-
parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb_consistent_backup.lock)", default='/tmp/mongodb_consistent_backup.lock', type=str)
61+
parser.add_argument("--lock-file", dest="lock_file", help="Location of lock file (default: /tmp/mongodb-consistent-backup.lock)", default='/tmp/mongodb-consistent-backup.lock', type=str)
6262
parser.add_argument("--sharding.balancer.wait_secs", dest="sharding.balancer.wait_secs", help="Maximum time to wait for balancer to stop, in seconds (default: 300)", default=300, type=int)
6363
parser.add_argument("--sharding.balancer.ping_secs", dest="sharding.balancer.ping_secs", help="Interval to check balancer state, in seconds (default: 3)", default=3, type=int)
6464
return self.makeParserLoadSubmodules(parser)

‎mongodb_consistent_backup/Main.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def __init__(self, prog_name="mongodb-consistent-backup"):
4444
self.backup_summary = {}
4545
self.manager = Manager()
4646
self.timer = Timer(self.manager)
47-
self.timer_name = "mongodb_consistent_backup.%s" % self.__class__.__name__
47+
self.timer_name = "%s.%s" % (self.program_name, self.__class__.__name__)
4848
self.backup_time = datetime.now().strftime("%Y%m%d_%H%M")
4949
self.logger = None
5050
self.current_log_file = None
@@ -460,5 +460,4 @@ def run(self):
460460
logging.info("Completed %s in %.2f sec" % (self.program_name, self.timer.duration(self.timer_name)))
461461

462462
self.logger.rotate()
463-
self.logger.close()
464463
self.release_lock()

‎mongodb_consistent_backup/Notify/Nsca/Nsca.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs):
3434
self.server_name = self.server
3535
self.server_port = 5667
3636
if ':' in self.server:
37-
self.server_name, self.server_port = self.server.split(":")
37+
self.server_name, port = self.server.split(":")
38+
self.server_port = int(port)
3839
self.server = "%s:%i" % (self.server_name, self.server_port)
3940

4041
try:

‎mongodb_consistent_backup/Oplog/Resolver/ResolverThread.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import logging
22
import os
3-
import sys
4-
53
# Skip bson in requirements , pymongo provides
64
# noinspection PyPackageRequirements
75
from bson import decode_file_iter
@@ -25,7 +23,7 @@ def cleanup(self):
2523
if 'tailed' in self.oplogs:
2624
self.oplogs['tailed'].close()
2725
del self.oplogs['tailed']
28-
if 'file' in self.tailed_opplog and os.path.isfile(self.tailed_oplog['file']):
26+
if 'file' in self.tailed_oplog and os.path.isfile(self.tailed_oplog['file']):
2927
os.remove(self.tailed_oplog['file'])
3028

3129
def run(self):
@@ -58,7 +56,6 @@ def run(self):
5856

5957
if self.exit_code == 0:
6058
logging.info("Applied %i oplog changes to %s oplog, end ts: %s" % (self.changes, self.uri, self.mongodump_oplog_h.last_ts()))
61-
sys.exit(self.exit_code)
6259

6360
def close(self):
6461
self.cleanup()

‎mongodb_consistent_backup/Pipeline/Stage.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,11 @@ def has_task(self):
5656
return False
5757

5858
def close(self):
59-
if self.has_task():
59+
if self.has_task() and not self.stopped:
6060
logging.debug("Calling close on backup stage %s with task %s" % (self.stage, self.task.capitalize()))
6161
self._task.close()
62+
self.running = False
63+
self.stopped = True
6264

6365
def is_compressed(self):
6466
if self.has_task() and hasattr(self._task, "is_compressed"):

0 commit comments

Comments
 (0)