moving to score-based secondary setup that is aware of hidden and priority flags

Author: Tim Vaillancourt

MongoBackup/ReplsetHandler.py

@@ -26,55 +26,85 @@ def __init__(self, host, port, user, password, authdb, max_lag_secs, retries=5):
     def close(self):
         return self.connection.close()
 
-    def get_rs_status(self):
+    def admin_command(self, admin_command):
         tries  = 0
         status = None
         while not status and tries < self.retries:
             try:
-                status = self.connection['admin'].command("replSetGetStatus")
+                status = self.connection['admin'].command(admin_command)
                 if not status:
                     raise e
             except Exception, e:
-                logging.error("Error running command 'replSetGetStatus': %s" % e)
+                logging.error("Error running command '%s': %s" % (admin_command, e))
                 tries += 1
                 sleep(1)
         if not status:
-            raise Exception, "Could not get output from command: 'replSetGetStatus' after %i retries!" % self.retries, None
+            raise Exception, "Could not get output from command: '%s' after %i retries!" % (admin_command, self.retries), None
         return status
 
+    def get_rs_status(self):
+        return self.admin_command('replSetGetStatus')
+
+    def get_rs_config(self):
+        return self.admin_command('replSetGetConfig')
+
     def find_desirable_secondary(self):
         rs_status    = self.get_rs_status()
+        rs_config    = self.get_rs_config()
         rs_name      = rs_status['set']
         quorum_count = ceil(len(rs_status['members']) / 2.0)
-        secondary    = None
-        primary      = None
+
+        primary = None
         for member in rs_status['members']:
-            if 'health' in member and member['health'] > 0:
-                logging.debug("Found %s: %s/%s with optime %s" % (
-                    member['stateStr'],
+            if member['stateStr'] == 'PRIMARY' and member['health'] > 0:
+                primary = {
+                    'host': member['name'],
+                    'optime': member['optimeDate']
+                }
+                logging.debug("Found PRIMARY: %s/%s with optime %s" % (
                     rs_name,
                     member['name'],
                     str(member['optime']['ts'])
                 ))
+        if primary is None:
+            logging.fatal("Unable to locate a PRIMARY member for replset %s, giving up" % rs_name)
+            raise Exception, "Unable to locate a PRIMARY member for replset %s, giving up" % rs_name, None
 
-                if member['stateStr'] == 'PRIMARY':
-                    primary = {
-                        'host': member['name'],
-                        'optime': member['optimeDate']
-                    }
-                elif member['stateStr'] == 'SECONDARY':
-                    if secondary is None or secondary['optime'] < member['optimeDate']:
+        secondary = None
+        for member in rs_status['members']:
+            if member['stateStr'] == 'SECONDARY' and member['health'] > 0:
+                log_data = {}
+                score    = 100
+
+                for member_config in rs_config['config']['members']:
+                    if member_config['host'] == member['name']:
+                        if 'hidden' in member_config and member_config['hidden'] == True:
+                            score += 20
+                            log_data['hidden'] = True
+                        if 'priority' in member_config:
+                            log_data['priority'] = member_config['priority']
+                            if member_config['priority'] > 1:
+                                score = score - member_config['priority']
+                        break
+
+                rep_lag = (mktime(primary['optime'].timetuple()) - mktime(member['optimeDate'].timetuple()))
+                score = score - rep_lag
+                if rep_lag < self.max_lag_secs:
+                    if secondary is None or score > secondary['score']:
                         secondary = {
-                            'replSet': rs_status['set'],
+                            'replSet': rs_name,
                             'count': 1 if secondary is None else secondary['count'] + 1,
                             'host': member['name'],
-                            'optime': member['optimeDate']
+                            'optime': member['optimeDate'],
+                            'score': score
                         }
+                    log_msg = "Found SECONDARY %s/%s" % (rs_name, member['name'])
+                else:
+                    log_msg = "Found SECONDARY %s/%s with too-high replication lag! Skipping" % (rs_name, member['name'])
 
-        if primary is None:
-            logging.fatal("Unable to locate a PRIMARY member for replset %s, giving up" % rs_name)
-            raise Exception, "Unable to locate a PRIMARY member for replset %s, giving up" % rs_name, None
-
+                log_data['optime'] = member['optime']['ts']
+                log_data['score']  = score
+                logging.debug("%s: %s" % (log_msg, str(log_data)))
         if secondary is None or (secondary['count'] + 1) < quorum_count:
             logging.fatal("Not enough secondaries in replset %s to take backup! Num replset members: %i, required quorum: %i" % (
                 rs_name,
@@ -83,12 +113,7 @@ def find_desirable_secondary(self):
             ))
             raise Exception, "Not enough secondaries in replset %s to safely take backup!" % rs_name, None
 
-        rep_lag = (mktime(primary['optime'].timetuple()) - mktime(secondary['optime'].timetuple()))
-        if rep_lag > self.max_lag_secs:
-            logging.fatal("No secondary found in replset %s within %s lag time!" % (rs_name, self.max_lag_secs))
-            raise Exception, "No secondary found in replset %s within %s lag time!" % (rs_name, self.max_lag_secs), None
-
-        logging.debug("Choosing SECONDARY %s for replica set %s" % (secondary['host'], rs_name))
+        logging.debug("Choosing SECONDARY %s for replica set %s (score: %i)" % (secondary['host'], rs_name, secondary['score']))
         return secondary