JAVA-334: too much exception logging from updater thread when a server is down

JAVA-347: in case all servers are down do not retry reads

Author: agirbal

src/main/com/mongodb/DBTCPConnector.java

@@ -277,11 +277,11 @@ public String getConnectPoint(){
 
     boolean _error( Throwable t, boolean slaveOk )
         throws MongoException {
-        if ( _allHosts != null ){
-            _logger.log( Level.WARNING , "replica set mode, switching master" , t );
+        if ( _rsStatus.hasServerUp() ){
+            // the replset has at least 1 server up, try to see if should switch master
             checkMaster( true , !slaveOk );
         }
-        return true;
+        return _rsStatus.hasServerUp();
     }
 
     class MyPort {
@@ -346,7 +346,7 @@ void done( DBPort p ){
         void error( DBPort p , Exception e ){
             p.close();
             _requestPort = null;
-            _logger.log( Level.SEVERE , "MyPort.error called" , e );            
+//            _logger.log( Level.SEVERE , "MyPort.error called" , e );
         }
 
         void requestEnsureConnection(){
@@ -437,7 +437,12 @@ void testMaster()
     }
 
     private boolean _set( ServerAddress addr ){
-        _masterPortPool = _portHolder.get( addr );
+        DBPortPool newPool = _portHolder.get( addr );
+        if (newPool == _masterPortPool)
+            return false;
+
+        _logger.log(Level.WARNING, "Master switching from " + (_masterPortPool != null ? _masterPortPool.getServerAddress() : "null") + " to " + addr);
+        _masterPortPool = newPool;
         return true;
     }
 

src/main/com/mongodb/ReplicaSetStatus.java

@@ -118,6 +118,16 @@ ServerAddress getASecondary(){
         return best._addr;
     }
 
+    boolean hasServerUp() {
+        for (int i = 0; i < _all.size(); i++) {
+            Node n = _all.get(i);
+            if (n._ok) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     class Node {
 
         Node( ServerAddress addr ){
@@ -151,10 +161,12 @@ synchronized void update(Set<Node> seenNodes){
                 _pingTime = _lastCheck - start;
 
                 if ( res == null ){
-                    _ok = false;
-                    return;
+                    throw new MongoInternalException("Invalid null value returned from isMaster");
                 }
 
+                if (!_ok) {
+                    _logger.log( Level.WARNING , "Server seen up: " + _addr );
+                }
                 _ok = true;
                 _isMaster = res.getBoolean( "ismaster" , false );
                 _isSecondary = res.getBoolean( "secondary" , false );
@@ -199,15 +211,12 @@ else if ( !_setName.equals( setName ) ){
                 }
 
             }
-            catch ( MongoException e ){
-                Throwable root = e;
-                if ( e.getCause() != null )
-                    root = e.getCause();
-                _logger.log( Level.FINE , "node down: " + _addr + " " + root );
-                _ok = false;
-            }
             catch ( Exception e ){
-                _logger.log( Level.SEVERE , "can't update node: " + _addr , e );
+                if (_ok == true) {
+                    _logger.log( Level.WARNING , "Server seen down: " + _addr, e );
+                } else if (Math.random() < 0.1) {
+                    _logger.log( Level.WARNING , "Server seen down: " + _addr );
+                }
                 _ok = false;
             }