perf: verify for float numbers during tokenization

Author: Alvaro-Kothe

pandas/_libs/include/pandas/parser/pd_parser.h

@@ -37,8 +37,8 @@ typedef struct {
   int (*parser_trim_buffers)(parser_t *);
   int (*tokenize_all_rows)(parser_t *, const char *);
   int (*tokenize_nrows)(parser_t *, size_t, const char *);
-  int64_t (*str_to_int64)(const char *, int64_t, int64_t, int *, char);
-  uint64_t (*str_to_uint64)(uint_state *, const char *, int64_t, uint64_t,
+  int64_t (*str_to_int64)(const char *, char, int64_t, int64_t, int *, char);
+  uint64_t (*str_to_uint64)(uint_state *, const char *, char, int64_t, uint64_t,
                             int *, char);
   double (*xstrtod)(const char *, char **, char, char, char, int, int *, int *);
   double (*precise_xstrtod)(const char *, char **, char, char, char, int, int *,
@@ -87,12 +87,14 @@ static PandasParser_CAPI *PandasParserAPI = NULL;
   PandasParserAPI->tokenize_all_rows((self), (encoding_errors))
 #define tokenize_nrows(self, nrows, encoding_errors)                           \
   PandasParserAPI->tokenize_nrows((self), (nrows), (encoding_errors))
-#define str_to_int64(p_item, int_min, int_max, error, t_sep)                   \
-  PandasParserAPI->str_to_int64((p_item), (int_min), (int_max), (error),       \
-                                (t_sep))
-#define str_to_uint64(state, p_item, int_max, uint_max, error, t_sep)          \
-  PandasParserAPI->str_to_uint64((state), (p_item), (int_max), (uint_max),     \
-                                 (error), (t_sep))
+#define str_to_int64(p_item, decimal_separator, int_min, int_max, error,       \
+                     t_sep)                                                    \
+  PandasParserAPI->str_to_int64((p_item), (decimal_separator), (int_min),      \
+                                (int_max), (error), (t_sep))
+#define str_to_uint64(state, p_item, decimal_separator, int_max, uint_max,     \
+                      error, t_sep)                                            \
+  PandasParserAPI->str_to_uint64((state), (p_item), (decimal_separator),       \
+                                 (int_max), (uint_max), (error), (t_sep))
 #define xstrtod(p, q, decimal, sci, tsep, skip_trailing, error, maybe_int)     \
   PandasParserAPI->xstrtod((p), (q), (decimal), (sci), (tsep),                 \
                            (skip_trailing), (error), (maybe_int))

pandas/_libs/include/pandas/parser/tokenizer.h

@@ -17,6 +17,7 @@ See LICENSE for the license
 #define ERROR_NO_DIGITS 1
 #define ERROR_OVERFLOW 2
 #define ERROR_INVALID_CHARS 3
+#define ERROR_IS_FLOAT 4
 
 #include <stdint.h>
 
@@ -208,10 +209,11 @@ void uint_state_init(uint_state *self);
 
 int uint64_conflict(uint_state *self);
 
-uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
+uint64_t str_to_uint64(uint_state *state, const char *p_item,
+                       char decimal_separator, int64_t int_max,
                        uint64_t uint_max, int *error, char tsep);
-int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
-                     int *error, char tsep);
+int64_t str_to_int64(const char *p_item, char decimal_separator,
+                     int64_t int_min, int64_t int_max, int *error, char tsep);
 double xstrtod(const char *p, char **q, char decimal, char sci, char tsep,
                int skip_trailing, int *error, int *maybe_int);
 double precise_xstrtod(const char *p, char **q, char decimal, char sci,

pandas/_libs/parsers.pyx

@@ -149,7 +149,7 @@ cdef extern from "pandas/parser/tokenizer.h":
         SKIP_LINE
         FINISHED
 
-    enum: ERROR_OVERFLOW
+    enum: ERROR_OVERFLOW, ERROR_IS_FLOAT
 
     ctypedef enum BadLineHandleMethod:
         ERROR,
@@ -281,10 +281,11 @@ cdef extern from "pandas/parser/pd_parser.h":
     int tokenize_all_rows(parser_t *self, const char *encoding_errors) nogil
     int tokenize_nrows(parser_t *self, size_t nrows, const char *encoding_errors) nogil
 
-    int64_t str_to_int64(char *p_item, int64_t int_min,
+    int64_t str_to_int64(char *p_item, char decimal_separator, int64_t int_min,
                          int64_t int_max, int *error, char tsep) nogil
-    uint64_t str_to_uint64(uint_state *state, char *p_item, int64_t int_max,
-                           uint64_t uint_max, int *error, char tsep) nogil
+    uint64_t str_to_uint64(uint_state *state, char *p_item, char decimal_separator,
+                           int64_t int_max, uint64_t uint_max,
+                           int *error, char tsep) nogil
 
     double xstrtod(const char *p, char **q, char decimal,
                    char sci, char tsep, int skip_trailing,
@@ -1070,21 +1071,28 @@ cdef class TextReader:
         else:
             col_res = None
             for dt in self.dtype_cast_order:
-                if (dt.kind in "iu" and
-                        self._column_has_float(i, start, end, na_filter, na_hashset)):
-                    continue
-
                 try:
                     col_res, na_count = self._convert_with_dtype(
                         dt, i, start, end, na_filter, 0, na_hashset, na_fset)
-                except ValueError:
-                    # This error is raised from trying to convert to uint64,
-                    # and we discover that we cannot convert to any numerical
-                    # dtype successfully. As a result, we leave the data
-                    # column AS IS with object dtype.
-                    col_res, na_count = self._convert_with_dtype(
-                        np.dtype("object"), i, start, end, 0,
-                        0, na_hashset, na_fset)
+                except ValueError as e:
+                    if str(e) == "Number is float":
+                        try:
+                            col_res, na_count = self._convert_with_dtype(
+                                np.dtype("float64"), i, start, end, 0,
+                                0, na_hashset, na_fset)
+                        except ValueError:
+                            col_res, na_count = self._convert_with_dtype(
+                                np.dtype("object"), i, start, end, 0,
+                                0, na_hashset, na_fset)
+
+                    else:
+                        # This error is raised from trying to convert to uint64,
+                        # and we discover that we cannot convert to any numerical
+                        # dtype successfully. As a result, we leave the data
+                        # column AS IS with object dtype.
+                        col_res, na_count = self._convert_with_dtype(
+                            np.dtype("object"), i, start, end, 0,
+                            0, na_hashset, na_fset)
                 except OverflowError:
                     try:
                         col_res, na_count = _try_pylong(self.parser, i, start,
@@ -1351,59 +1359,6 @@ cdef class TextReader:
             else:
                 return None
 
-    cdef bint _column_has_float(self, Py_ssize_t col,
-                                int64_t start, int64_t end,
-                                bint na_filter, kh_str_starts_t *na_hashset):
-        """Check if the column contains any float number."""
-        cdef:
-            Py_ssize_t i, j, lines = end - start
-            coliter_t it
-            const char *word = NULL
-            const char *ignored_chars = " +-"
-            const char *digits = "0123456789"
-            const char *float_indicating_chars = "eE"
-            char null_byte = 0
-
-        coliter_setup(&it, self.parser, col, start)
-
-        for i in range(lines):
-            COLITER_NEXT(it, word)
-
-            if na_filter and kh_get_str_starts_item(na_hashset, word):
-                continue
-
-            found_first_digit = False
-            j = 0
-            while word[j] != null_byte:
-                if word[j] == self.parser.decimal:
-                    return True
-                elif not found_first_digit and word[j] in ignored_chars:
-                    # no-op
-                    pass
-                elif not found_first_digit and word[j] not in digits:
-                    # word isn't numeric
-                    return False
-                elif not found_first_digit and word[j] in digits:
-                    found_first_digit = True
-                elif word[j] in float_indicating_chars:
-                    # preceding chars indicates numeric and
-                    # current char indicates float
-                    return True
-                elif word[j] not in digits:
-                    # previous characters indicates numeric
-                    # current character shows otherwise
-                    return False
-                elif word[j] in digits:
-                    # no-op
-                    pass
-                else:
-                    raise AssertionError(
-                            f"Unhandled case {word[j]=} {found_first_digit=}"
-                            )
-                j += 1
-
-        return False
-
 # Factor out code common to TextReader.__dealloc__ and TextReader.close
 # It cannot be a class method, since calling self.close() in __dealloc__
 # which causes a class attribute lookup and violates best practices
@@ -1822,6 +1777,8 @@ cdef _try_uint64(parser_t *parser, int64_t col,
         if error == ERROR_OVERFLOW:
             # Can't get the word variable
             raise OverflowError("Overflow")
+        elif error == ERROR_IS_FLOAT:
+            raise ValueError("Number is float")
         return None
 
     if uint64_conflict(&state):
@@ -1855,14 +1812,14 @@ cdef int _try_uint64_nogil(parser_t *parser, int64_t col,
                 data[i] = 0
                 continue
 
-            data[i] = str_to_uint64(state, word, INT64_MAX, UINT64_MAX,
+            data[i] = str_to_uint64(state, word, parser.decimal, INT64_MAX, UINT64_MAX,
                                     &error, parser.thousands)
             if error != 0:
                 return error
     else:
         for i in range(lines):
             COLITER_NEXT(it, word)
-            data[i] = str_to_uint64(state, word, INT64_MAX, UINT64_MAX,
+            data[i] = str_to_uint64(state, word, parser.decimal, INT64_MAX, UINT64_MAX,
                                     &error, parser.thousands)
             if error != 0:
                 return error
@@ -1892,6 +1849,8 @@ cdef _try_int64(parser_t *parser, int64_t col,
         if error == ERROR_OVERFLOW:
             # Can't get the word variable
             raise OverflowError("Overflow")
+        elif error == ERROR_IS_FLOAT:
+            raise ValueError("Number is float")
         return None, None
 
     return result, na_count
@@ -1920,14 +1879,14 @@ cdef int _try_int64_nogil(parser_t *parser, int64_t col,
                 data[i] = NA
                 continue
 
-            data[i] = str_to_int64(word, INT64_MIN, INT64_MAX,
+            data[i] = str_to_int64(word, parser.decimal, INT64_MIN, INT64_MAX,
                                    &error, parser.thousands)
             if error != 0:
                 return error
     else:
         for i in range(lines):
             COLITER_NEXT(it, word)
-            data[i] = str_to_int64(word, INT64_MIN, INT64_MAX,
+            data[i] = str_to_int64(word, parser.decimal, INT64_MIN, INT64_MAX,
                                    &error, parser.thousands)
             if error != 0:
                 return error

pandas/_libs/src/parser/tokenizer.c

@@ -1834,8 +1834,8 @@ int uint64_conflict(uint_state *self) {
   return self->seen_uint && (self->seen_sint || self->seen_null);
 }
 
-int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
-                     int *error, char tsep) {
+int64_t str_to_int64(const char *p_item, char decimal_separator,
+                     int64_t int_min, int64_t int_max, int *error, char tsep) {
   const char *p = p_item;
   // Skip leading spaces.
   while (isspace_ascii(*p)) {
@@ -1879,7 +1879,7 @@ int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
           d = *++p;
         } else {
           *error = ERROR_OVERFLOW;
-          return 0;
+          break;
         }
       }
     } else {
@@ -1890,7 +1890,7 @@ int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
           d = *++p;
         } else {
           *error = ERROR_OVERFLOW;
-          return 0;
+          break;
         }
       }
     }
@@ -1917,7 +1917,7 @@ int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
 
         } else {
           *error = ERROR_OVERFLOW;
-          return 0;
+          break;
         }
       }
     } else {
@@ -1929,28 +1929,46 @@ int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
 
         } else {
           *error = ERROR_OVERFLOW;
-          return 0;
+          break;
         }
       }
     }
   }
 
+  if (*error == ERROR_OVERFLOW) {
+    // advance digits
+    while (*p != '\0' && isdigit_ascii(*p)) {
+      p++;
+    }
+
+    // check if is float
+    if (*p == decimal_separator || *p == 'e' || *p == 'E') {
+      *error = ERROR_IS_FLOAT;
+    }
+    return 0;
+  }
+
   // Skip trailing spaces.
   while (isspace_ascii(*p)) {
     ++p;
   }
 
   // Did we use up all the characters?
   if (*p) {
-    *error = ERROR_INVALID_CHARS;
+    if (*p == decimal_separator || *p == 'e' || *p == 'E') {
+      *error = ERROR_IS_FLOAT;
+    } else {
+      *error = ERROR_INVALID_CHARS;
+    }
     return 0;
   }
 
   *error = 0;
   return number;
 }
 
-uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
+uint64_t str_to_uint64(uint_state *state, const char *p_item,
+                       char decimal_separator, int64_t int_max,
                        uint64_t uint_max, int *error, char tsep) {
   const char *p = p_item;
   // Skip leading spaces.
@@ -1997,7 +2015,7 @@ uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
 
       } else {
         *error = ERROR_OVERFLOW;
-        return 0;
+        break;
       }
     }
   } else {
@@ -2009,19 +2027,36 @@ uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
 
       } else {
         *error = ERROR_OVERFLOW;
-        return 0;
+        break;
       }
     }
   }
 
+  if (*error == ERROR_OVERFLOW) {
+    // advance digits
+    while (*p != '\0' && isdigit_ascii(*p)) {
+      p++;
+    }
+
+    // check if is float
+    if (*p == decimal_separator || *p == 'e' || *p == 'E') {
+      *error = ERROR_IS_FLOAT;
+    }
+    return 0;
+  }
+
   // Skip trailing spaces.
   while (isspace_ascii(*p)) {
     ++p;
   }
 
   // Did we use up all the characters?
   if (*p) {
-    *error = ERROR_INVALID_CHARS;
+    if (*p == decimal_separator || *p == 'e' || *p == 'E') {
+      *error = ERROR_IS_FLOAT;
+    } else {
+      *error = ERROR_INVALID_CHARS;
+    }
     return 0;
   }