Skip to content

Commit bf7b319

Browse files
timvaillancourttimvaillancourt
committed
Squash commit of s3_upload_fixes_v1
1 parent 6468a72 commit bf7b319

File tree

10 files changed

+540
-242
lines changed

10 files changed

+540
-242
lines changed

‎.travis.yml‎

Lines changed: 51 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -3,93 +3,70 @@ python:
33
- "2.7"
44
cache:
55
pip: true
6-
directories:
7-
- $TRAVIS_BUILD_DIR/bin
8-
- $TRAVIS_BUILD_DIR/build/docker
9-
- $TRAVIS_BUILD_DIR/build/rpm/RPMS
6+
directories:
7+
- $TRAVIS_BUILD_DIR/docker
108
- $TRAVIS_BUILD_DIR/tmp/pex
119
- $TRAVIS_BUILD_DIR/tmp/pip
1210
services:
1311
- docker
14-
install: true
15-
stages:
16-
# - flake8
17-
- build
18-
# - test-cluster-3.4
19-
# - test-cluster-3.2
20-
# - test-replset-3.4
21-
# - test-replset-3.2
22-
# - test-replset-3.0
23-
# - test-archive-none
24-
# - test-archive-zbackup
25-
- name: build-debian
26-
if: branch =~ ^\d+\.\d+\.\d+$ AND tag =~ ^\d+\.\d+\.\d+$
27-
- name: deploy-dockerhub
28-
if: branch =~ ^\d+\.\d+\.\d+$ AND tag =~ ^\d+\.\d+\.\d+$
29-
- name: deploy-github
30-
if: branch =~ ^\d+\.\d+\.\d+$ AND tag =~ ^\d+\.\d+\.\d+$
12+
install: 'pip install virtualenv'
3113
jobs:
3214
include:
33-
# - stage: flake8
34-
# script:
35-
# - pip install flake8
36-
# - make flake8
15+
- stage: flake8
16+
script:
17+
- pip install flake8
18+
- make flake8
3719
- stage: build
3820
script:
3921
- make docker
4022
- docker run --rm -it mongodb_consistent_backup:latest --version
41-
- mkdir -p $TRAVIS_BUILD_DIR/build/docker
42-
- docker save mongodb_consistent_backup:latest >$TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
43-
# - stage: test-cluster-3.4
44-
# script:
45-
# - docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
46-
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.4
47-
# - stage: test-replset-3.4
48-
# script:
49-
# - docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
50-
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4
51-
# - stage: test-cluster-3.2
23+
- mkdir -p $TRAVIS_BUILD_DIR/docker
24+
- docker save mongodb_consistent_backup:latest >$TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
25+
- stage: test-cluster-3.4
26+
script:
27+
- docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
28+
- $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.4
29+
- stage: test-replset-3.4
30+
script:
31+
- docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
32+
- $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4
33+
- stage: test-cluster-3.2
34+
script:
35+
- docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
36+
- $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.2
37+
# - stage: test-cluster-3.2-sccc
5238
# script:
53-
# - docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
39+
# - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
5440
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.2
55-
# - stage: test-replset-3.2
41+
- stage: test-replset-3.2
42+
script:
43+
- docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
44+
- $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.2
45+
# - stage: test-cluster-3.0
5646
# script:
57-
# - docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
58-
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.2
59-
# - stage: test-replset-3.0
47+
# - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
48+
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-cluster.sh 3.0 SCCC
49+
- stage: test-replset-3.0
50+
script:
51+
- docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
52+
- $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.0
53+
- stage: test-archive-none
54+
script:
55+
- docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
56+
- $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --archive.method=none
57+
- stage: test-archive-zbackup
58+
script:
59+
- docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
60+
- $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --archive.method=zbackup
61+
# - stage: test-upload-gs
6062
# script:
61-
# - docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
62-
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.0
63-
# - stage: test-archive-none
63+
# - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
64+
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --upload.method=gs
65+
# - stage: test-upload-rsync
6466
# script:
65-
# - docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
66-
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --archive.method=none
67-
# - stage: test-archive-zbackup
67+
# - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
68+
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --upload.method=rsync
69+
# - stage: test-upload-s3
6870
# script:
69-
# - docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar
70-
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --archive.method=zbackup
71-
- stage: build-debian
72-
script:
73-
- make debian8
74-
- make debian9
75-
- stage: deploy-dockerhub
76-
script: true
77-
deploy:
78-
provider: script
79-
skip_cleanup: true
80-
script: docker load -i $TRAVIS_BUILD_DIR/build/docker/mongodb_consistent_backup.tar && docker tag mongodb_consistent_backup $DOCKERHUB_PATH:latest && docker tag mongodb_consistent_backup:latest $DOCKERHUB_PATH:$TRAVIS_TAG && docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD && docker push $DOCKERHUB_PATH:latest && docker push $DOCKERHUB_PATH:$TRAVIS_TAG
81-
on:
82-
tags: true
83-
- stage: deploy-github
84-
script: MCB_VERSION=$(cat $TRAVIS_BUILD_DIR/VERSION)
85-
deploy:
86-
provider: releases
87-
skip_cleanup: true
88-
api_key: $GITHUB_OAUTH_TOKEN
89-
file:
90-
- $TRAVIS_BUILD_DIR/build/rpm/RPMS/x86_64/mongodb_consistent_backup-$MCB_VERSION-1.el7.centos.x86_64.rpm"
91-
- $TRAVIS_BUILD_DIR/bin/mongodb-consistent-backup.el7.centos.x86_64
92-
- $TRAVIS_BUILD_DIR/bin/mongodb-consistent-backup.debian8.x86_64
93-
- $TRAVIS_BUILD_DIR/bin/mongodb-consistent-backup.debian9.x86_64
94-
on:
95-
tags: true
71+
# - docker load -i $TRAVIS_BUILD_DIR/docker/mongodb_consistent_backup.tar
72+
# - $TRAVIS_BUILD_DIR/scripts/travis-ci/run-replset.sh 3.4 --upload.method=s3

‎mongodb_consistent_backup/Common/Util.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import socket
22

33
from dateutil import parser
4+
from hashlib import md5
45
from select import select
56

67
from mongodb_consistent_backup.Errors import OperationError
@@ -52,3 +53,11 @@ def wait_popen(process, stderr_callback, stdout_callback):
5253
except Exception, e:
5354
raise e
5455
return True
56+
57+
58+
def file_md5hash(file_path, blocksize=65536):
59+
md5hash = md5()
60+
with open(file_path, "rb") as f:
61+
for block in iter(lambda: f.read(blocksize), b""):
62+
md5hash.update(block)
63+
return md5hash.hexdigest()

‎mongodb_consistent_backup/Upload/Gs/Gs.py‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
import logging
22
import os
33

4-
54
from copy_reg import pickle
65
from multiprocessing import Pool
76
from types import MethodType
87

98
from mongodb_consistent_backup.Errors import OperationError
109
from mongodb_consistent_backup.Pipeline import Task
10+
from mongodb_consistent_backup.Upload.Util import get_upload_files
11+
1112
from GsUploadThread import GsUploadThread
1213

1314

@@ -41,17 +42,6 @@ def close(self):
4142
self._pool.terminate()
4243
self.stopped = True
4344

44-
def get_backup_files(self, base_dir=None, files=[]):
45-
if not base_dir:
46-
base_dir = self.backup_dir
47-
for child in os.listdir(base_dir):
48-
path = os.path.join(base_dir, child)
49-
if os.path.isfile(path):
50-
files.append(path)
51-
elif os.path.isdir(path):
52-
self.get_backup_files(path, files)
53-
return files
54-
5545
def run(self):
5646
if not os.path.isdir(self.backup_dir):
5747
logging.error("The source directory: %s does not exist or is not a directory! Skipping Google Cloud Storage upload!" % self.backup_dir)
@@ -60,7 +50,7 @@ def run(self):
6050
self.running = True
6151
self.timer.start(self.timer_name)
6252
logging.info("Uploading %s to Google Cloud Storage (bucket=%s, threads=%i)" % (self.base_dir, self.bucket, self.threads()))
63-
for file_path in self.get_backup_files():
53+
for file_path in get_upload_files():
6454
gs_path = os.path.relpath(file_path, self.backup_location)
6555
self._pool.apply_async(GsUploadThread(
6656
self.backup_dir,

‎mongodb_consistent_backup/Upload/Gs/GsUploadThread.py‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import boto
2-
import hashlib
32
import logging
43
import os
54

5+
from mongodb_consistent_backup.Common.Util import file_md5hash
66
from mongodb_consistent_backup.Errors import OperationError
77

88

@@ -34,7 +34,7 @@ def configure(self):
3434
def get_uri(self):
3535
return boto.storage_uri(self.path, 'gs')
3636

37-
def exists(self):
37+
def gs_exists(self):
3838
try:
3939
self.metadata()
4040
return True
@@ -52,13 +52,6 @@ def gs_md5hash(self):
5252
if hasattr(key, 'etag'):
5353
return key.etag.strip('"\'')
5454

55-
def file_md5hash(self, blocksize=65536):
56-
md5 = hashlib.md5()
57-
with open(self.file_path, "rb") as f:
58-
for block in iter(lambda: f.read(blocksize), b""):
59-
md5.update(block)
60-
return md5.hexdigest()
61-
6255
def success(self):
6356
if self.remove_uploaded and not self.file_path.startswith(os.path.join(self.backup_dir, self.meta_data_dir)):
6457
logging.debug("Removing successfully uploaded file: %s" % self.file_path)
@@ -68,8 +61,9 @@ def run(self):
6861
f = None
6962
try:
7063
self.configure()
71-
if self.exists():
72-
if self.gs_md5hash() and self.file_md5hash() == self.gs_md5hash():
64+
if self.gs_exists():
65+
gs_md5hash = self.gs_md5hash()
66+
if gs_md5hash and file_md5hash(self.file_path) == gs_md5hash:
7367
logging.debug("Path %s already exists with the same checksum (%s), skipping" % (self.path, self.gs_md5hash()))
7468
return
7569
logging.debug("Path %s checksum and local checksum differ, re-uploading" % self.path)

0 commit comments

Comments
 (0)