Merge remote-tracking branch 'upstream/master' into location_and_name_defaults_update

Author: timvaillancourt

Dockerfile

@@ -1,9 +1,13 @@
 FROM centos:centos7
-ARG RELEASE
 MAINTAINER Tim Vaillancourt <tim.vaillancourt@percona.com>
+
 RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \
 	yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all
-RUN curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/$RELEASE/mongodb-consistent-backup.el7.centos.x86_64 && \
-	chmod +x /usr/bin/mongodb-consistent-backup
+
+ADD build/rpm/RPMS/x86_64/mongodb_consistent_backup*.el*.centos.x86_64.rpm /
+RUN yum localinstall -y /mongodb_consistent_backup*.el*.centos.x86_64.rpm && \
+	yum clean all && rm -f /mongodb_consistent_backup*.el*.centos.x86_64.rpm
+
+USER mongodb_consistent_backup
 ENTRYPOINT ["mongodb-consistent-backup"]
 CMD ["--help"]

Makefile

@@ -2,22 +2,26 @@
 # 
 
 NAME=mongodb_consistent_backup
+BIN_NAME?=mongodb-consistent-backup
 VERSION=$(shell cat VERSION | cut -d- -f1)
+GIT_COMMIT?=$(shell git show 2>/dev/null | awk 'NR==1{print $$2}')
 PREFIX?=/usr/local
+ARCH?=x86_64
 BASEDIR?=$(DESTDIR)$(PREFIX)
 BINDIR?=$(BASEDIR)/bin
 SHAREDIR?=$(BASEDIR)/share
 DOCKER_TAG?="$(NAME):$(VERSION)"
+DOCKER_BASE_IMAGE?=$(shell awk '/FROM/{print $$2}' Dockerfile)
+MAKE_DIR=$(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 
+all: bin/$(BIN_NAME)
 
-all: bin/mongodb-consistent-backup
+bin/$(BIN_NAME): setup.py requirements.txt README.rst VERSION scripts/build.sh $(NAME)/*.py $(NAME)/*/*.py $(NAME)/*/*/*.py
+	BIN_NAME=$(BIN_NAME) GIT_COMMIT=$(GIT_COMMIT) PYTHON_BIN=$(PYTHON_BIN) VIRTUALENV_BIN=$(VIRTUALENV_BIN) bash scripts/build.sh
 
-bin/mongodb-consistent-backup: setup.py requirements.txt README.rst VERSION scripts/build.sh $(NAME)/*.py $(NAME)/*/*.py $(NAME)/*/*/*.py
-	PYTHON_BIN=$(PYTHON_BIN) VIRTUALENV_BIN=$(VIRTUALENV_BIN) bash scripts/build.sh
-
-install: bin/mongodb-consistent-backup
+install: bin/$(BIN_NAME)
 	mkdir -p $(BINDIR) $(SHAREDIR)/$(NAME) || true
-	install -m 0755 bin/mongodb-consistent-backup $(BINDIR)/mongodb-consistent-backup
+	install -m 0755 bin/$(BIN_NAME) $(BINDIR)/mongodb-consistent-backup
 	install -m 0644 conf/mongodb-consistent-backup.example.conf $(SHAREDIR)/$(NAME)/example.conf
 	install -m 0644 LICENSE $(SHAREDIR)/$(NAME)/LICENSE
 	install -m 0644 README.rst $(SHAREDIR)/$(NAME)/README.rst
@@ -26,21 +30,90 @@ flake8:
 	# Ignore long-lines and space-aligned = and : for now
 	flake8 --ignore E221,E241,E501 $(PWD)/$(NAME)
 
+rpm: bin/$(BIN_NAME)
+	mkdir -p $(MAKE_DIR)/build/rpm/SOURCES
+	cp -f $(MAKE_DIR)/{LICENSE,README.rst} build/rpm/SOURCES
+	cp -f $(MAKE_DIR)/bin/$(BIN_NAME) build/rpm/SOURCES/mongodb-consistent-backup
+	cp -f $(MAKE_DIR)/conf/mongodb-consistent-backup.example.conf build/rpm/SOURCES/mongodb-consistent-backup.conf
+	rpmbuild -D "_topdir $(MAKE_DIR)/build/rpm" -D "version $(VERSION)" -bb $(MAKE_DIR)/scripts/$(NAME).spec
+
 uninstall:
 	rm -f $(BINDIR)/mongodb-consistent-backup
 	rm -rf $(SHAREDIR)/$(NAME)
 
-rpm: bin/mongodb-consistent-backup
-	rm -rf build/rpm 2>/dev/null || true
-	mkdir -p build/rpm/SOURCES
-	cp -f $(PWD)/{LICENSE,README.rst} build/rpm/SOURCES
-	cp -f $(PWD)/bin/mongodb-consistent-backup build/rpm/SOURCES/mongodb-consistent-backup
-	cp -f $(PWD)/conf/mongodb-consistent-backup.example.conf build/rpm/SOURCES/mongodb-consistent-backup.conf
-	rpmbuild -D "_topdir $(PWD)/build/rpm" -D "version $(VERSION)" -bb scripts/$(NAME).spec
+# Build CentOS7 RPM (in Docker)
+build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-1.el7.centos.$(ARCH).rpm:
+	mkdir -p $(MAKE_DIR)/build/rpm/RPMS/$(ARCH)
+	docker run --rm \
+		-v "$(MAKE_DIR)/bin:/src/bin:Z" \
+		-v "$(MAKE_DIR)/conf:/src/conf:Z" \
+		-v "$(MAKE_DIR)/mongodb_consistent_backup:/src/mongodb_consistent_backup:Z" \
+		-v "$(MAKE_DIR)/scripts:/src/scripts:Z" \
+		-v "$(MAKE_DIR)/tmp/pip:/src/tmp/pip:Z" \
+		-v "$(MAKE_DIR)/setup.py:/src/setup.py:Z" \
+		-v "$(MAKE_DIR)/requirements.txt:/src/requirements.txt:Z" \
+		-v "$(MAKE_DIR)/Makefile:/src/Makefile:Z" \
+		-v "$(MAKE_DIR)/README.rst:/src/README.rst:Z" \
+		-v "$(MAKE_DIR)/LICENSE:/src/LICENSE:Z" \
+		-v "$(MAKE_DIR)/VERSION:/src/VERSION:Z" \
+		-v "$(MAKE_DIR)/build/rpm/RPMS/$(ARCH):/src/build/rpm/RPMS/$(ARCH):Z" \
+		-it centos:centos7 \
+		/bin/bash -c "yum install -y python-devel python-virtualenv gcc make libffi-devel openssl-devel rpm-build && \
+			make -C /src GIT_COMMIT=$(GIT_COMMIT) BIN_NAME=mongodb-consistent-backup.el7.centos.$(ARCH) rpm && \
+			/src/bin/mongodb-consistent-backup.el7.centos.$(ARCH) --version"
+
+centos7: build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-1.el7.centos.$(ARCH).rpm
+
+# Build Debian8 Binary (in Docker - .deb package soon!)
+bin/mongodb-consistent-backup.debian8.$(ARCH):
+	docker run --rm \
+		-v "$(MAKE_DIR)/bin:/src/bin:Z" \
+		-v "$(MAKE_DIR)/conf:/src/conf:Z" \
+		-v "$(MAKE_DIR)/mongodb_consistent_backup:/src/mongodb_consistent_backup:Z" \
+		-v "$(MAKE_DIR)/scripts:/src/scripts:Z" \
+		-v "$(MAKE_DIR)/tmp/pip:/src/tmp/pip:Z" \
+		-v "$(MAKE_DIR)/setup.py:/src/setup.py:Z" \
+		-v "$(MAKE_DIR)/requirements.txt:/src/requirements.txt:Z" \
+		-v "$(MAKE_DIR)/Makefile:/src/Makefile:Z" \
+		-v "$(MAKE_DIR)/README.rst:/src/README.rst:Z" \
+		-v "$(MAKE_DIR)/LICENSE:/src/LICENSE:Z" \
+		-v "$(MAKE_DIR)/VERSION:/src/VERSION:Z" \
+		-it debian:jessie \
+		/bin/bash -c "apt-get update && apt-get install -y python2.7-minimal python2.7-dev python-virtualenv gcc make libffi-dev libssl-dev && \
+			make -C /src GIT_COMMIT=$(GIT_COMMIT) BIN_NAME=mongodb-consistent-backup.debian8.$(ARCH).tmp && \
+			mv -vf /src/bin/mongodb-consistent-backup.debian8.$(ARCH).tmp /src/bin/mongodb-consistent-backup.debian8.$(ARCH) && \
+			/src/bin/mongodb-consistent-backup.debian8.$(ARCH) --version"
 
-docker: bin/mongodb-consistent-backup
-	docker build --no-cache --tag $(DOCKER_TAG) --build-arg "RELEASE=$(VERSION)" .
+debian8: bin/mongodb-consistent-backup.debian8.$(ARCH)
+
+# Build Debian9 Binary (in Docker - .deb package soon!)
+bin/mongodb-consistent-backup.debian9.$(ARCH):
+	docker run --rm \
+		-v "$(MAKE_DIR)/bin:/src/bin:Z" \
+		-v "$(MAKE_DIR)/conf:/src/conf:Z" \
+		-v "$(MAKE_DIR)/mongodb_consistent_backup:/src/mongodb_consistent_backup:Z" \
+		-v "$(MAKE_DIR)/scripts:/src/scripts:Z" \
+		-v "$(MAKE_DIR)/tmp/pip:/src/tmp/pip:Z" \
+		-v "$(MAKE_DIR)/setup.py:/src/setup.py:Z" \
+		-v "$(MAKE_DIR)/requirements.txt:/src/requirements.txt:Z" \
+		-v "$(MAKE_DIR)/Makefile:/src/Makefile:Z" \
+		-v "$(MAKE_DIR)/README.rst:/src/README.rst:Z" \
+		-v "$(MAKE_DIR)/LICENSE:/src/LICENSE:Z" \
+		-v "$(MAKE_DIR)/VERSION:/src/VERSION:Z" \
+		-it debian:stretch \
+		/bin/bash -c "apt-get update && apt-get install -y python2.7-minimal python2.7-dev python-virtualenv gcc make libffi-dev libssl-dev && \
+			make -C /src GIT_COMMIT=$(GIT_COMMIT) BIN_NAME=mongodb-consistent-backup.debian9.$(ARCH).tmp && \
+			mv -vf /src/bin/mongodb-consistent-backup.debian9.$(ARCH).tmp /src/bin/mongodb-consistent-backup.debian9.$(ARCH) && \
+			/src/bin/mongodb-consistent-backup.debian9.$(ARCH) --version"
+
+debian9: bin/mongodb-consistent-backup.debian9.$(ARCH)
+
+docker: build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-1.el7.centos.$(ARCH).rpm
+	docker build --no-cache --tag $(DOCKER_TAG) .
 	docker tag $(DOCKER_TAG) $(NAME):latest
+	docker run --rm -it $(DOCKER_TAG) --version
+
+release: centos7 debian8 debian9 docker
 
 clean:
 	rm -rf bin build $(NAME).egg-info tmp 2>/dev/null

README.rst

@@ -31,9 +31,11 @@ Features
    archiving method (*optional*)
 -  `AWS S3 <https://aws.amazon.com/s3/>`__ Secure Multipart backup uploads (*optional*)
 -  `Google Cloud Storage <https://cloud.google.com/storage/>`__ Secure backup uploads (*optional*)
+-  Rsync (over SSH) secure backup uploads (*optional*)
 -  `Nagios NSCA <https://sourceforge.net/p/nagios/nsca>`__ push
    notification support (*optional*)
 -  Modular backup, archiving, upload and notification components
+-  Support for MongoDB Authentication and SSL database connections
 -  Multi-threaded, single executable
 -  Auto-scales to number of available CPUs by default
 
@@ -77,6 +79,7 @@ To build an CentOS/RedHat RPM of the tool *(recommended)*:
 ::
 
     $ cd /path/to/mongodb_consistent_backup
+    $ sudo yum install -y rpm-build
     $ make rpm
 
 To build and install from source *(to default '/usr/local/bin/mongodb-consistent-backup')*:
@@ -148,25 +151,40 @@ The backups are `mongorestore <https://docs.mongodb.com/manual/reference/program
     ...
     $ mongorestore --host mongod12.example.com --port 27017 -u admin -p 123456 --oplogReplay --dir /var/lib/mongodb-consistent-backup/default/20170424_0000/rs0/dump
 
-Run as Docker Container (Experimental)
+Run as Docker Container
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Note: you need to use persistent volumes to store backups and/or config files long-term when using Docker. Data in Docker containers is destroyed when the container is deleted. See `scripts/docker-persistent.sh <scripts/docker-persistent.sh>`__ and `scripts/docker-persistent.example.conf <scripts/docker-persistent.example.conf>`__ as an example/demo of how to implement persistence.
+To persist logs, configs and backup data 3 directories should be mapped to be inside the Docker containter.
+
+The 'docker run' command -v/--volume flags in the examples below map container paths to paths on your Docker host. The example below assumes there is a path on the Docker host named *'/data/mongobackup'* with *'data'*, *'conf'* and *'logs'* subdirectories mapped to inside the container. Replace any instance of *'/data/mongobackup'* below to a different path if necessary.
+
+*Note: store a copy of your mongodb-consistent-backup.conf in the 'conf' directory and pass it's container path as the --config= flag if you wish to use config files.*
 
 **Via Docker Hub**
 
 ::
 
-    $ docker run -i timvaillancourt/mongodb_consistent_backup <mongodb_consistent_backup-flags>
+    $ mkdir -p /data/mongobackup/{conf,data,logs}
+    $ cp -f /path/to/mongodb-consistent-backup.conf /data/mongobackup/conf
+    $ docker run -it \
+        -v "/data/mongobackup/conf:/conf:Z" \
+        -v "/data/mongobackup/data:/var/lib/mongodb-consistent-backup:Z" \
+        -v "/data/mongobackup/logs:/var/log/mongodb-consistent-backup:Z" \
+      timvaillancourt/mongodb_consistent_backup:latest --config=/conf/mongodb-consistent-backup.conf
 
 **Build and Run Docker Image**
 
 ::
 
     $ cd /path/to/mongodb_consistent_backup
     $ make docker
-    $ docker run -t mongodb_consistent_backup <mongodb_consistent_backup-flags>
-
+    $ mkdir -p /data/mongobackup/{conf,data,logs}
+    $ cp -f /path/to/mongodb-consistent-backup.conf /data/mongobackup/conf
+    $ docker run -it \
+        -v "/data/mongobackup/conf:/conf:Z" \
+        -v "/data/mongobackup/data:/var/lib/mongodb-consistent-backup:Z" \
+        -v "/data/mongobackup/logs:/var/log/mongodb-consistent-backup:Z" \
+      mongodb_consistent_backup --config=/conf/mongodb-consistent-backup.conf
 
 ZBackup Archiving (Optional)
 ~~~~~~~

VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

conf/mongodb-consistent-backup.example.conf

@@ -4,15 +4,21 @@ production:
   #username: [auth username] (default: none)
   #password: [auth password] (default: none)
   #authdb: [auth database]   (default: admin)
+  #ssl:
+  #  enabled: [true|false]  (default: false)
+  #  insecure: [true|false] (default: false)
+  #  ca_file: [path]
+  #  crl_file: [path]
+  #  client_cert_file: [path]
   log_dir: /var/log/mongodb-consistent-backup
   backup:
     method: mongodump
     name: default
     location: /var/lib/mongodb-consistent-backup
   #  mongodump:
   #    binary: [path]                (default: /usr/bin/mongodump)
-  #    compression: [auto|none|gzip] (default: auto - enable gzip if supported)
-  #    threads: [1-16]               (default: auto-generated - shards/cpu)
+  #    compression: [auto|none|gzip] (default: auto = enable gzip if supported)
+  #    threads: [1-16]               (default: auto-generated, shards/cpu)
   #replication:
   #  max_lag_secs: [1+]        (default: 10)
   #  min_priority: [0-999]     (default: 0)
@@ -23,18 +29,19 @@ production:
   #    wait_secs: [1+] (default: 300)
   #    ping_secs: [1+] (default: 3)
   #oplog:
-  #  compression: [none|gzip] (default: gzip - if gzip is used by backup stage)
+  #  compression: [none|gzip] (default: gzip, if used by backup stage)
   #  flush:
-  #    max_docs: 100
-  #    max_secs: 1
-  #  resolver_threads: [1+]   (default: 2 per CPU)
+  #    max_docs: [1+] (default: 100)
+  #    max_secs: [1+] (default: 1)
+  #  resolver:
+  #    threads: [1+]  (default: 2 per CPU)
   #  tailer:
   #    enabled: true
   #    status_interval: 30
   archive:
     method: tar
   #  tar:
-  #    compression: [none|gzip] (default: gzip - none if backup is already compressed)
+  #    compression: [none|gzip] (default: gzip, none if backup already compressed)
   #    threads: [1+]            (default: 1 per CPU)
   #  zbackup:
   #    binary: [path]        (default: /usr/bin/zbackup)
@@ -52,21 +59,26 @@ production:
   upload:
     method: none
   #  remove_uploaded: [true|false] (default: false)
+  #  retries: [1+]           (default: 5)
+  #  threads: [1+]           (default: 4)
   #  gs:
   #    project_id: [Google Cloud Project ID]
-  #    access_key: [Google Cloud Storage Access Key]
-  #    secret_key: [Google Cloud Storage Secret Key]
+  #    access_key: [Google Cloud Storage Interoperability API Access Key]
+  #    secret_key: [Google Cloud Storage Interoperability API Secret Key]
   #    bucket_name: [Google Cloud Storage Bucket Name]
   #    bucket_prefix: [prefix] (default: /)
-  #    threads: [1+]           (default: 1 per CPU)
+  #  rsync:
+  #    path: [Rsync Destination Path]
+  #    user: [SSH Username]
+  #    host: [SSH Hostname/IP]
+  #    port: [SSH Port Number] (default: 22)
+  #    delete: [true|false]    (default: false)
   #  s3:
   #    region: [AWS S3 Region] (default: us-east-1)
   #    access_key: [AWS S3 Access Key]
   #    secret_key: [AWS S3 Secret Key]
   #    bucket_name: [AWS S3 Bucket Name]
   #    bucket_prefix: [prefix] (default: /)
-  #    threads: [1+]           (default: 1 per CPU)
   #    chunk_size_mb: [1+]     (default: 50)
   #    secure: [true|false]    (default: true)
-  #    retries: [1+]           (default: 5)
   #    acl: [acl_str]          (default: none)

mongodb_consistent_backup/Archive/Archive.py

@@ -1,3 +1,5 @@
+from mongodb_consistent_backup.Archive.Tar import Tar  # NOQA
+from mongodb_consistent_backup.Archive.Zbackup import Zbackup  # NOQA
 from mongodb_consistent_backup.Pipeline import Stage
 
 

mongodb_consistent_backup/Backup/Mongodump/Mongodump.py

@@ -1,6 +1,5 @@
 import os
 import logging
-import sys
 
 from math import floor
 from subprocess import check_output
@@ -57,22 +56,21 @@ def parse_mongodump_version(self):
         raise OperationError("Could not parse mongodump --version output!")
 
     def choose_compression(self):
-        if self.can_gzip():
+        if self.can_compress():
             if self.compression() == 'auto':
                 logging.info("Mongodump binary supports gzip compression, auto-enabling gzip compression")
                 self.compression('gzip')
         elif self.compression() == 'gzip':
             raise OperationError("mongodump gzip compression requested on binary that does not support gzip!")
 
-    def can_gzip(self):
+    def can_compress(self):
         if os.path.isfile(self.binary) and os.access(self.binary, os.X_OK):
-            logging.debug("Mongodump binary supports gzip")
+            logging.debug("Mongodump binary supports gzip compression")
             if tuple("3.2.0".split(".")) <= tuple(self.version.split(".")):
                 return True
             return False
         else:
-            logging.fatal("Cannot find or execute the mongodump binary file %s!" % self.binary)
-            sys.exit(1)
+            raise OperationError("Cannot find or execute the mongodump binary file %s!" % self.binary)
 
     def summary(self):
         return self._summary