fix: allow read_csv with python engine read large integers as int

Author: Alvaro-Kothe

pandas/_libs/lib.pyx

@@ -1386,6 +1386,7 @@ cdef class Seen:
         bint nan_             # seen_np.nan
         bint uint_            # seen_uint (unsigned integer)
         bint sint_            # seen_sint (signed integer)
+        bint overflow_        # seen_overflow
         bint float_           # seen_float
         bint object_          # seen_object
         bint complex_         # seen_complex
@@ -1414,6 +1415,7 @@ cdef class Seen:
         self.nan_ = False
         self.uint_ = False
         self.sint_ = False
+        self.overflow_ = False
         self.float_ = False
         self.object_ = False
         self.complex_ = False
@@ -2379,6 +2381,9 @@ def maybe_convert_numeric(
         ndarray[uint64_t, ndim=1] uints = cnp.PyArray_EMPTY(
             1, values.shape, cnp.NPY_UINT64, 0
         )
+        ndarray[object, ndim=1] pyints = cnp.PyArray_EMPTY(
+            1, values.shape, cnp.NPY_OBJECT, 0
+        )
         ndarray[uint8_t, ndim=1] bools = cnp.PyArray_EMPTY(
             1, values.shape, cnp.NPY_UINT8, 0
         )
@@ -2476,6 +2481,7 @@ def maybe_convert_numeric(
 
                 if maybe_int:
                     as_int = int(val)
+                    pyints[i] = as_int
 
                     if as_int in na_values:
                         mask[i] = 1
@@ -2490,7 +2496,7 @@ def maybe_convert_numeric(
                             if seen.coerce_numeric:
                                 seen.float_ = True
                             else:
-                                raise ValueError("Integer out of range.")
+                                seen.overflow_ = True
                         else:
                             if as_int >= 0:
                                 uints[i] = as_int
@@ -2529,11 +2535,15 @@ def maybe_convert_numeric(
         return (floats, None)
     elif seen.int_:
         if seen.null_ and convert_to_masked_nullable:
-            if seen.uint_:
+            if seen.overflow_:
+                return (pyints, mask.view(np.bool_))
+            elif seen.uint_:
                 return (uints, mask.view(np.bool_))
             else:
                 return (ints, mask.view(np.bool_))
-        if seen.uint_:
+        if seen.overflow_:
+            return (pyints, None)
+        elif seen.uint_:
             return (uints, None)
         else:
             return (ints, None)

pandas/io/parsers/base_parser.py

@@ -521,7 +521,11 @@ def _infer_types(
             if values.dtype == np.object_:
                 na_count = parsers.sanitize_objects(values, na_values)
 
-        if result.dtype == np.object_ and try_num_bool:
+        if (
+            result.dtype == np.object_
+            and try_num_bool
+            and (len(result) == 0 or not isinstance(result[0], int))
+        ):
             result, bool_mask = libops.maybe_convert_bool(
                 np.asarray(values),
                 true_values=self.true_values,

pandas/tests/io/parser/common/test_ints.py

@@ -144,11 +144,6 @@ def test_int64_overflow(all_parsers, conv, request):
         if parser.engine == "pyarrow":
             mark = pytest.mark.xfail(reason="parses to float64")
             request.applymarker(mark)
-        elif parser.engine == "python":
-            mark = pytest.mark.xfail(
-                reason="TODO: Python engine reads bigint as string"
-            )
-            request.applymarker(mark)
 
         result = parser.read_csv(StringIO(data))
         expected = DataFrame(
@@ -206,9 +201,6 @@ def test_outside_int64_uint64_range(all_parsers, val, request):
     # These numbers fall just outside the int64-uint64
     # range, so they should be parsed as object.
     parser = all_parsers
-    if parser.engine == "python":
-        mark = pytest.mark.xfail(reason="TODO: Python engine reads bigint as string")
-        request.applymarker(mark)
 
     result = parser.read_csv(StringIO(str(val)), header=None)