use BSON.encode to determine payload

Author: Jibola

django_mongodb_backend/compiler.py

@@ -1,8 +1,7 @@
 import itertools
-import sys
 from collections import defaultdict
 
-from bson import SON, json_util
+from bson import SON, encode, json_util
 from django.core.exceptions import EmptyResultSet, FieldError, FullResultSet
 from django.db import IntegrityError, NotSupportedError
 from django.db.models import Count
@@ -335,9 +334,10 @@ def cursor_iter(self, cursor, _, columns):
             if len(chunk) == chunk_size:
                 if i == chunk_size - 1:  # First chunk complete
                     # Using current row as representation, approximate
-                    # how many rows can fit in a 16MB payload (MongoDB batch_size max)
-                    # then set that as the new chunk size.
-                    chunk_size = MAX_BATCH_SIZE_MB // sys.getsizeof(row)
+                    # how many rows can fit in a 16MB payload
+                    # (MongoDB batch_size max), then set that as the new size.
+                    approx_chunk_size = MAX_BATCH_SIZE_MB // len(encode(row))
+                    chunk_size = max(chunk_size, approx_chunk_size)
                 yield chunk
                 chunk = []
         yield chunk