[JAVA-289]: detect max bson size from servers. Update whenever master changes. Fetch on 1st op in single server mode.

Author: agirbal

src/main/com/mongodb/Bytes.java

@@ -49,11 +49,11 @@ public class Bytes extends BSON {
     /** Little-endian */
     public static final ByteOrder ORDER = ByteOrder.LITTLE_ENDIAN;
 
-    /** this size is used to prevent insertion of objects that are too large for db */
-    static final int MAX_OBJECT_SIZE = 1024 * 1024 * 32;
+    /** this size is set low to 4MB, but just serves as safe default */
+    static final int MAX_OBJECT_SIZE = 1024 * 1024 * 4;
 
-    /** target size of an insert batch */
-    static final int BATCH_INSERT_SIZE = 1024 * 1024 * 16;
+    /** default target size of an insert batch */
+    static final int BATCH_INSERT_SIZE = 1024 * 1024 * 8;
 
     static final int CONNECTIONS_PER_HOST = Integer.parseInt( System.getProperty( "MONGO.POOLSIZE" , "10" ) );
 

src/main/com/mongodb/DBApiLayer.java

@@ -211,6 +211,7 @@ protected WriteResult insert(DBObject[] arr, boolean shouldApply , com.mongodb.W
             WriteResult last = null;
 
             int cur = 0;
+            int maxsize = _mongo.getMaxBsonObjectSize();
             while ( cur < arr.length ){
                 OutMessage om = new OutMessage( _mongo , 2002 );
 
@@ -219,14 +220,10 @@ protected WriteResult insert(DBObject[] arr, boolean shouldApply , com.mongodb.W
 
                 for ( ; cur<arr.length; cur++ ){
                     DBObject o = arr[cur];
-                    int sz = om.putObject( o );
-                    // server is better suited to decide on object size
-                    // indeed driver may be talking to dbs with different limits
-                    // also max size is tough to catch on updates, so here it's just an extra check
-                    if ( sz > Bytes.MAX_OBJECT_SIZE )
-                        throw new IllegalArgumentException( "object too big: " + sz );
-                    
-                    if ( om.size() > Bytes.BATCH_INSERT_SIZE ){
+                    om.putObject( o );
+
+                    // limit for batch insert is 4 x maxbson on server, use 2 x to be safe
+                    if ( om.size() > 2 * maxsize ){
                         cur++;
                         break;
                     }

src/main/com/mongodb/DBPort.java

@@ -141,24 +141,17 @@ synchronized CommandResult runCommand( DB db , DBObject cmd ) throws IOException
         return (CommandResult)res;
     }
 
-    synchronized DBObject findOne( String ns , DBObject q ){
+    synchronized DBObject findOne( String ns , DBObject q ) throws IOException{
         OutMessage msg = OutMessage.query( null , 0 , ns , 0 , -1 , q , null );
-        
-        try {
-            Response res = go( msg , null , true );
-            if ( res.size() == 0 )
-                return null;
-            if ( res.size() > 1 )
-                throw new MongoInternalException( "something is wrong.  size:" + res.size() );
-            return res.get(0);
-        }
-        catch ( IOException ioe ){
-            throw new MongoException.Network( "DBPort.findOne failed" , ioe );
-        }
-        
+        Response res = go( msg , null , true );
+        if ( res.size() == 0 )
+            return null;
+        if ( res.size() > 1 )
+            throw new MongoInternalException( "something is wrong.  size:" + res.size() );
+        return res.get(0);
     }
 
-    synchronized CommandResult runCommand( String db , DBObject cmd ) {
+    synchronized CommandResult runCommand( String db , DBObject cmd ) throws IOException {
         DBObject res = findOne( db + ".$cmd" , cmd );
         if ( res == null )
             throw new MongoInternalException( "something is wrong, no command result" );

src/main/com/mongodb/DBTCPConnector.java

@@ -400,11 +400,41 @@ void checkMaster( boolean force , boolean failIfNoMaster )
                 }
                 else {
                     _set( n._addr );
+                    maxBsonObjectSize = _rsStatus.getMaxBsonObjectSize();
                 }
             }
+        } else {
+            // single server, may have to obtain max bson size
+            if (maxBsonObjectSize == 0)
+                    maxBsonObjectSize = fetchMaxBsonObjectSize();
+        }
+    }
+
+    /**
+     * Fetches the maximum size for a BSON object from the current master server
+     * @return the size, or 0 if it could not be obtained
+     */
+    int fetchMaxBsonObjectSize() {
+        if (_masterPortPool == null)
+            return 0;
+        DBPort port = _masterPortPool.get();
+        try {
+            CommandResult res = port.runCommand(_mongo.getDB("admin"), new BasicDBObject("isMaster", 1));
+            // max size was added in 1.8
+            if (res.containsField("maxBsonObjectSize")) {
+                maxBsonObjectSize = ((Integer) res.get("maxBsonObjectSize")).intValue();
+            } else {
+                maxBsonObjectSize = Bytes.MAX_OBJECT_SIZE;
+            }
+        } catch (Exception e) {
+            _logger.log(Level.WARNING, null, e);
+        } finally {
+            port.getPool().done(port);
         }
+        return maxBsonObjectSize;
     }
 
+
     void testMaster()
         throws MongoException {
 
@@ -469,13 +499,23 @@ public boolean isOpen(){
         return ! _closed;
     }
 
+    /**
+     * Gets the maximum size for a BSON object supported by the current master server.
+     * Note that this value may change over time depending on which server is master.
+     * @return the maximum size, or 0 if not obtained from servers yet.
+     */
+    public int getMaxBsonObjectSize() {
+        return maxBsonObjectSize;
+    }
+
     private Mongo _mongo;
 //    private ServerAddress _curMaster;
     private DBPortPool _masterPortPool;
     private DBPortPool.Holder _portHolder;
     private final List<ServerAddress> _allHosts;
     private final ReplicaSetStatus _rsStatus;
     private boolean _closed = false;
+    private int maxBsonObjectSize = 0;
 
     private ThreadLocal<MyPort> _myPort = new ThreadLocal<MyPort>(){
         protected MyPort initialValue(){

src/main/com/mongodb/Mongo.java

@@ -516,22 +516,15 @@ void _applyMongoOptions() {
 
     /**
      * Gets the maximum size for a BSON object supported by the current master server.
-     * Note that this method may make a request to the server to obtain the size.
-     * @throws MongoException
-     * @return the size supported, or 0 if it cannot be determined.
+     * Note that this value may change over time depending on which server is master.
+     * If the size is not known yet, a request may be sent to the master server
+     * @return the maximum size
      */
     public int getMaxBsonObjectSize() {
-        int size = 0;
-        if (_connector.getReplicaSetStatus() != null)
-            size = _connector.getReplicaSetStatus().getMaxBsonObjectSize();
-        if (size == 0) {
-            // need to fetch size
-            DB db = getDB("admin");
-            CommandResult res = db.command("isMaster");
-            if (res.containsField("maxBsonObjectSize"))
-                size = ((Integer)res.get( "maxBsonObjectSize" )).intValue();
-        }
-        return size;
+        int maxsize = _connector.getMaxBsonObjectSize();
+        if (maxsize == 0)
+            maxsize = _connector.fetchMaxBsonObjectSize();
+        return maxsize > 0 ? maxsize : Bytes.MAX_OBJECT_SIZE;
     }
 
     final ServerAddress _addr;

src/main/com/mongodb/OutMessage.java

@@ -183,6 +183,20 @@ int getId(){
         return _id;
     }
 
+    @Override
+    public int putObject(BSONObject o) {
+        // check max size
+        int sz = super.putObject(o);
+        if (_mongo != null) {
+            int maxsize = _mongo.getConnector().getMaxBsonObjectSize();
+            maxsize = Math.max(maxsize, Bytes.MAX_OBJECT_SIZE);
+            if (sz > maxsize) {
+                throw new MongoInternalException("DBObject of size " + sz + " is over Max BSON size " + _mongo.getMaxBsonObjectSize());
+            }
+        }
+        return sz;
+    }
+
     private Mongo _mongo;
     private PoolOutputBuffer _buffer;
     private int _id;

src/main/com/mongodb/ReplicaSetStatus.java

@@ -170,8 +170,13 @@ synchronized void update(Set<Node> seenNodes){
                     }
                 }
 
-                if (_isMaster && res.containsField("maxBsonObjectSize"))
-                    _maxBsonObjectSize = ((Integer)res.get( "maxBsonObjectSize" )).intValue();
+                if (_isMaster ) {
+                    // max size was added in 1.8
+                    if (res.containsField("maxBsonObjectSize"))
+                        maxBsonObjectSize = ((Integer)res.get( "maxBsonObjectSize" )).intValue();
+                    else
+                        maxBsonObjectSize = Bytes.MAX_OBJECT_SIZE;
+                }
 
             }
             catch ( MongoException e ){
@@ -392,15 +397,20 @@ void close(){
         _closed = true;
     }
 
+    /**
+     * Gets the maximum size for a BSON object supported by the current master server.
+     * Note that this value may change over time depending on which server is master.
+     * @return the maximum size, or 0 if not obtained from servers yet.
+     */
     public int getMaxBsonObjectSize() {
-        return _maxBsonObjectSize;
+        return maxBsonObjectSize;
     }
 
     final List<Node> _all;
     Updater _updater;
     Mongo _mongo;
     String _setName = null; // null until init
-    int _maxBsonObjectSize = 0;
+    int maxBsonObjectSize = 0;
     Logger _logger = _rootLogger; // will get changed to use set name once its found
 
     String _lastPrimarySignal;

src/test/com/mongodb/JavaClientTest.java

@@ -535,7 +535,7 @@ public void testObjectIdCompat2(){
     @Test
     public void testLargeBulkInsert(){
         // max size should be obtained from server
-        int maxObjSize = 16 * 1024 * 1024;
+        int maxObjSize = _mongo.getMaxBsonObjectSize();
         DBCollection c = _db.getCollection( "largebulk" );
         c.drop();
         String s = "asdasdasd";
@@ -562,7 +562,7 @@ public void testLargeBulkInsert(){
             c.save( new BasicDBObject( "foo" , s ) );
             worked = true;
         }
-        catch ( IllegalArgumentException ie ){}
+        catch ( MongoException ie ){}
         assertFalse( worked );
 
         assertEquals( num , c.find().count() );