Merge pull request #212 from timvaillancourt/1.2.0-upload-thread-retries

1.2.0: Make 'threads' and 'retries' vars upload-wide (#190)

Author: dbmurphy

conf/mongodb-consistent-backup.example.conf

@@ -59,29 +59,26 @@ production:
   upload:
     method: none
   #  remove_uploaded: [true|false] (default: false)
+  #  retries: [1+]           (default: 5)
+  #  threads: [1+]           (default: 4)
   #  gs:
   #    project_id: [Google Cloud Project ID]
-  #    access_key: [Google Cloud Storage Access Key]
-  #    secret_key: [Google Cloud Storage Secret Key]
+  #    access_key: [Google Cloud Storage Interoperability API Access Key]
+  #    secret_key: [Google Cloud Storage Interoperability API Secret Key]
   #    bucket_name: [Google Cloud Storage Bucket Name]
   #    bucket_prefix: [prefix] (default: /)
-  #    threads: [1+]           (default: 4)
   #  rsync:
   #    path: [Rsync Destination Path]
   #    user: [SSH Username]
   #    host: [SSH Hostname/IP]
   #    port: [SSH Port Number] (default: 22)
-  #    delete:  [true|false]   (default: false)
-  #    threads: [1+]           (default: 4)
-  #    retries: [1+]           (default: 5)
+  #    delete: [true|false]    (default: false)
   #  s3:
   #    region: [AWS S3 Region] (default: us-east-1)
   #    access_key: [AWS S3 Access Key]
   #    secret_key: [AWS S3 Secret Key]
   #    bucket_name: [AWS S3 Bucket Name]
   #    bucket_prefix: [prefix] (default: /)
-  #    threads: [1+]           (default: 1 per CPU)
   #    chunk_size_mb: [1+]     (default: 50)
   #    secure: [true|false]    (default: true)
-  #    retries: [1+]           (default: 5)
   #    acl: [acl_str]          (default: none)

mongodb_consistent_backup/Upload/Gs/__init__.py

@@ -2,9 +2,9 @@
 
 
 def config(parser):
-    parser.add_argument("--upload.gs.project_id", dest="upload.gs.project_id", help="Google Cloud Storage Uploader Project ID (required for GS upload)", type=str)
-    parser.add_argument("--upload.gs.access_key", dest="upload.gs.access_key", help="Google Cloud Storage Uploader Access Key (required for GS upload)", type=str)
-    parser.add_argument("--upload.gs.secret_key", dest="upload.gs.secret_key", help="Google Cloud Storage Uploader Secret Key (required for GS upload)", type=str)
-    parser.add_argument("--upload.gs.bucket_name", dest="upload.gs.bucket_name", help="Google Cloud Storage Uploader destination bucket name", type=str)
-    parser.add_argument("--upload.gs.bucket_prefix", dest="upload.gs.bucket_prefix", help="Google Cloud Storage Uploader destination bucket path prefix", type=str)
+    parser.add_argument("--upload.gs.project_id", dest="upload.gs.project_id", help="Google Cloud Storage Project ID (required for GS upload)", type=str)
+    parser.add_argument("--upload.gs.access_key", dest="upload.gs.access_key", help="Google Cloud Storage Interoperability API Access Key (required for GS upload)", type=str)
+    parser.add_argument("--upload.gs.secret_key", dest="upload.gs.secret_key", help="Google Cloud Storage Interoperability API Secret Key (required for GS upload)", type=str)
+    parser.add_argument("--upload.gs.bucket_name", dest="upload.gs.bucket_name", help="Google Cloud Storage destination bucket name", type=str)
+    parser.add_argument("--upload.gs.bucket_prefix", dest="upload.gs.bucket_prefix", help="Google Cloud Storage destination bucket path prefix", type=str)
     return parser

mongodb_consistent_backup/Upload/Rsync/Rsync.py

@@ -31,13 +31,13 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs):
         self.backup_location = self.config.backup.location
         self.backup_name     = self.config.backup.name
         self.remove_uploaded = self.config.upload.remove_uploaded
+        self.retries         = self.config.upload.retries
+        self.thread_count    = self.config.upload.threads
         self.rsync_path      = self.config.upload.rsync.path
         self.rsync_user      = self.config.upload.rsync.user
         self.rsync_host      = self.config.upload.rsync.host
         self.rsync_port      = self.config.upload.rsync.port
         self.rsync_ssh_key   = self.config.upload.rsync.ssh_key
-        self.retries         = self.config.upload.rsync.retries
-        self.thread_count    = self.config.upload.rsync.threads
         self.rsync_binary    = "rsync"
 
         self.rsync_flags   = ["--archive", "--compress"]

mongodb_consistent_backup/Upload/Rsync/__init__.py

@@ -7,6 +7,4 @@ def config(parser):
     parser.add_argument("--upload.rsync.host", dest="upload.rsync.host", help="Rsync upload SSH hostname/IP", default=None, type=str)
     parser.add_argument("--upload.rsync.port", dest="upload.rsync.port", help="Rsync upload SSH port number (default: 22)", default=22, type=int)
     parser.add_argument("--upload.rsync.ssh_key", dest="upload.rsync.ssh_key", help="Rsync upload SSH key path", default=None, type=str)
-    parser.add_argument("--upload.rsync.retries", dest="upload.rsync.retries", help="Rsync upload retries (default: 5)", default=5, type=int)
-    parser.add_argument("--upload.rsync.threads", dest="upload.rsync.threads", help="Rsync upload threads (default: 4)", default=4, type=int)
     return parser