Merge branch 'main' into api-filter-select

Author: jbrockmendel

doc/source/whatsnew/v3.0.0.rst

@@ -717,6 +717,7 @@ Other Deprecations
 - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.unstack` and :meth:`DataFrame.unstack` (:issue:`12189`, :issue:`53868`)
 - Deprecated :meth:`Series.filter` and :meth:`DataFrame.filter`, renaming these to ``select`` (:issue:`26642`)
 - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.shift` and :meth:`DataFrame.shift` (:issue:`53802`)
+- Deprecated option "future.no_silent_downcasting", as it is no longer used. In a future version accessing this option will raise (:issue:`59502`)
 - Deprecated slicing on a :class:`Series` or :class:`DataFrame` with a :class:`DatetimeIndex` using a ``datetime.date`` object, explicitly cast to :class:`Timestamp` instead (:issue:`35830`)
 
 .. ---------------------------------------------------------------------------
@@ -1013,12 +1014,13 @@ Strings
 ^^^^^^^
 - Bug in :meth:`Series.str.zfill` raising ``AttributeError`` for :class:`ArrowDtype` (:issue:`61485`)
 - Bug in :meth:`Series.value_counts` would not respect ``sort=False`` for series having ``string`` dtype (:issue:`55224`)
+- Bug in multiplication with a :class:`StringDtype` incorrectly allowing multiplying by bools; explicitly cast to integers instead (:issue:`62595`)
 
 Interval
 ^^^^^^^^
 - :meth:`Index.is_monotonic_decreasing`, :meth:`Index.is_monotonic_increasing`, and :meth:`Index.is_unique` could incorrectly be ``False`` for an ``Index`` created from a slice of another ``Index``. (:issue:`57911`)
+- Bug in :class:`Index`, :class:`Series`, :class:`DataFrame` constructors when given a sequence of :class:`Interval` subclass objects casting them to :class:`Interval` (:issue:`46945`)
 - Bug in :func:`interval_range` where start and end numeric types were always cast to 64 bit (:issue:`57268`)
--
 
 Indexing
 ^^^^^^^^

pandas/_libs/lib.pyx

@@ -2255,7 +2255,8 @@ cpdef bint is_interval_array(ndarray values):
     for i in range(n):
         val = values[i]
 
-        if isinstance(val, Interval):
+        if type(val) is Interval:
+            # GH#46945 catch Interval exactly, excluding subclasses
             if closed is None:
                 closed = val.closed
                 numeric = (

pandas/_libs/src/parser/tokenizer.c

@@ -23,10 +23,15 @@ GitHub. See Python Software Foundation License and BSD licenses for these.
 #include <float.h>
 #include <math.h>
 #include <stdbool.h>
+#include <stdlib.h>
 
 #include "pandas/portable.h"
 #include "pandas/vendored/klib/khash.h" // for kh_int64_t, kh_destroy_int64
 
+// Arrow256 allows up to 76 decimal digits.
+// We rounded up to the next power of 2.
+#define PROCESSED_WORD_CAPACITY 128
+
 void coliter_setup(coliter_t *self, parser_t *parser, int64_t i,
                    int64_t start) {
   // column i, starting at 0
@@ -1834,6 +1839,39 @@ int uint64_conflict(uint_state *self) {
   return self->seen_uint && (self->seen_sint || self->seen_null);
 }
 
+/* Copy a string without `char_to_remove` into `output`.
+ */
+static int copy_string_without_char(char output[PROCESSED_WORD_CAPACITY],
+                                    const char *str, size_t str_len,
+                                    char char_to_remove) {
+  const char *left = str;
+  const char *end_ptr = str + str_len;
+  size_t bytes_written = 0;
+
+  while (left < end_ptr) {
+    const size_t remaining_bytes_to_read = end_ptr - left;
+    const char *right = memchr(left, char_to_remove, remaining_bytes_to_read);
+
+    if (!right) {
+      // If it doesn't find the char to remove, just copy until EOS.
+      right = end_ptr;
+    }
+
+    const size_t chunk_size = right - left;
+
+    if (chunk_size + bytes_written >= PROCESSED_WORD_CAPACITY) {
+      return -1;
+    }
+    memcpy(&output[bytes_written], left, chunk_size);
+    bytes_written += chunk_size;
+
+    left = right + 1;
+  }
+
+  output[bytes_written] = '\0';
+  return 0;
+}
+
 int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
                      int *error, char tsep) {
   const char *p = p_item;
@@ -1843,105 +1881,45 @@ int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
   }
 
   // Handle sign.
-  const bool isneg = *p == '-' ? true : false;
+  const bool has_sign = *p == '-' || *p == '+';
   // Handle sign.
-  if (isneg || (*p == '+')) {
-    p++;
-  }
+  const char *digit_start = has_sign ? p + 1 : p;
 
   // Check that there is a first digit.
-  if (!isdigit_ascii(*p)) {
+  if (!isdigit_ascii(*digit_start)) {
     // Error...
     *error = ERROR_NO_DIGITS;
     return 0;
   }
 
-  int64_t number = 0;
-  if (isneg) {
-    // If number is greater than pre_min, at least one more digit
-    // can be processed without overflowing.
-    int dig_pre_min = -(int_min % 10);
-    int64_t pre_min = int_min / 10;
-
-    // Process the digits.
-    char d = *p;
-    if (tsep != '\0') {
-      while (1) {
-        if (d == tsep) {
-          d = *++p;
-          continue;
-        } else if (!isdigit_ascii(d)) {
-          break;
-        }
-        if ((number > pre_min) ||
-            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
-          number = number * 10 - (d - '0');
-          d = *++p;
-        } else {
-          *error = ERROR_OVERFLOW;
-          return 0;
-        }
-      }
-    } else {
-      while (isdigit_ascii(d)) {
-        if ((number > pre_min) ||
-            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
-          number = number * 10 - (d - '0');
-          d = *++p;
-        } else {
-          *error = ERROR_OVERFLOW;
-          return 0;
-        }
-      }
+  char buffer[PROCESSED_WORD_CAPACITY];
+  const size_t str_len = strlen(p);
+  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
+    const int status = copy_string_without_char(buffer, p, str_len, tsep);
+    if (status != 0) {
+      // Word is too big, probably will cause an overflow
+      *error = ERROR_OVERFLOW;
+      return 0;
     }
-  } else {
-    // If number is less than pre_max, at least one more digit
-    // can be processed without overflowing.
-    int64_t pre_max = int_max / 10;
-    int dig_pre_max = int_max % 10;
-
-    // Process the digits.
-    char d = *p;
-    if (tsep != '\0') {
-      while (1) {
-        if (d == tsep) {
-          d = *++p;
-          continue;
-        } else if (!isdigit_ascii(d)) {
-          break;
-        }
-        if ((number < pre_max) ||
-            ((number == pre_max) && (d - '0' <= dig_pre_max))) {
-          number = number * 10 + (d - '0');
-          d = *++p;
+    p = buffer;
+  }
 
-        } else {
-          *error = ERROR_OVERFLOW;
-          return 0;
-        }
-      }
-    } else {
-      while (isdigit_ascii(d)) {
-        if ((number < pre_max) ||
-            ((number == pre_max) && (d - '0' <= dig_pre_max))) {
-          number = number * 10 + (d - '0');
-          d = *++p;
+  char *endptr;
+  int64_t number = strtoll(p, &endptr, 10);
 
-        } else {
-          *error = ERROR_OVERFLOW;
-          return 0;
-        }
-      }
-    }
+  if (errno == ERANGE || number > int_max || number < int_min) {
+    *error = ERROR_OVERFLOW;
+    errno = 0;
+    return 0;
   }
 
   // Skip trailing spaces.
-  while (isspace_ascii(*p)) {
-    ++p;
+  while (isspace_ascii(*endptr)) {
+    ++endptr;
   }
 
   // Did we use up all the characters?
-  if (*p) {
+  if (*endptr) {
     *error = ERROR_INVALID_CHARS;
     return 0;
   }
@@ -1974,53 +1952,34 @@ uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
     return 0;
   }
 
-  // If number is less than pre_max, at least one more digit
-  // can be processed without overflowing.
-  //
-  // Process the digits.
-  uint64_t number = 0;
-  const uint64_t pre_max = uint_max / 10;
-  const uint64_t dig_pre_max = uint_max % 10;
-  char d = *p;
-  if (tsep != '\0') {
-    while (1) {
-      if (d == tsep) {
-        d = *++p;
-        continue;
-      } else if (!isdigit_ascii(d)) {
-        break;
-      }
-      if ((number < pre_max) ||
-          ((number == pre_max) && ((uint64_t)(d - '0') <= dig_pre_max))) {
-        number = number * 10 + (d - '0');
-        d = *++p;
-
-      } else {
-        *error = ERROR_OVERFLOW;
-        return 0;
-      }
+  char buffer[PROCESSED_WORD_CAPACITY];
+  const size_t str_len = strlen(p);
+  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
+    const int status = copy_string_without_char(buffer, p, str_len, tsep);
+    if (status != 0) {
+      // Word is too big, probably will cause an overflow
+      *error = ERROR_OVERFLOW;
+      return 0;
     }
-  } else {
-    while (isdigit_ascii(d)) {
-      if ((number < pre_max) ||
-          ((number == pre_max) && ((uint64_t)(d - '0') <= dig_pre_max))) {
-        number = number * 10 + (d - '0');
-        d = *++p;
+    p = buffer;
+  }
 
-      } else {
-        *error = ERROR_OVERFLOW;
-        return 0;
-      }
-    }
+  char *endptr;
+  uint64_t number = strtoull(p, &endptr, 10);
+
+  if (errno == ERANGE || number > uint_max) {
+    *error = ERROR_OVERFLOW;
+    errno = 0;
+    return 0;
   }
 
   // Skip trailing spaces.
-  while (isspace_ascii(*p)) {
-    ++p;
+  while (isspace_ascii(*endptr)) {
+    ++endptr;
   }
 
   // Did we use up all the characters?
-  if (*p) {
+  if (*endptr) {
     *error = ERROR_INVALID_CHARS;
     return 0;
   }

pandas/conftest.py

@@ -689,7 +689,9 @@ def _create_mi_with_dt64tz_level():
     "categorical": CategoricalIndex(list("abcd") * 2),
     "interval": IntervalIndex.from_breaks(np.linspace(0, 100, num=11)),
     "empty": Index([]),
-    "tuples": MultiIndex.from_tuples(zip(["foo", "bar", "baz"], [1, 2, 3])),
+    "tuples": MultiIndex.from_tuples(
+        zip(["foo", "bar", "baz"], [1, 2, 3], strict=True)
+    ),
     "mi-with-dt64tz-level": _create_mi_with_dt64tz_level(),
     "multi": _create_multiindex(),
     "repeats": Index([0, 0, 1, 1, 2, 2]),
@@ -1874,7 +1876,9 @@ def any_numeric_dtype(request):
     ("period", [Period(2013), pd.NaT, Period(2018)]),
     ("interval", [Interval(0, 1), np.nan, Interval(0, 2)]),
 ]
-ids, _ = zip(*_any_skipna_inferred_dtype)  # use inferred type as fixture-id
+ids = [
+    pair[0] for pair in _any_skipna_inferred_dtype
+]  # use inferred type as fixture-id
 
 
 @pytest.fixture(params=_any_skipna_inferred_dtype, ids=ids)

pandas/core/arrays/string_.py

@@ -1113,6 +1113,16 @@ def _cmp_method(self, other, op):
                 other = np.asarray(other)
             other = other[valid]
 
+        other_dtype = getattr(other, "dtype", None)
+        if op.__name__.strip("_") in ["mul", "rmul"] and (
+            lib.is_bool(other) or lib.is_np_dtype(other_dtype, "b")
+        ):
+            # GH#62595
+            raise TypeError(
+                "Cannot multiply StringArray by bools. "
+                "Explicitly cast to integers instead."
+            )
+
         if op.__name__ in ops.ARITHMETIC_BINOPS:
             result = np.empty_like(self._ndarray, dtype="object")
             result[mask] = self.dtype.na_value

pandas/core/config_init.py

@@ -28,6 +28,8 @@
     is_text,
 )
 
+from pandas.errors import Pandas4Warning
+
 # compute
 
 use_bottleneck_doc = """
@@ -899,10 +901,10 @@ def register_converter_cb(key: str) -> None:
     cf.register_option(
         "no_silent_downcasting",
         False,
-        "Whether to opt-in to the future behavior which will *not* silently "
-        "downcast results from Series and DataFrame `where`, `mask`, and `clip` "
-        "methods. "
-        "Silent downcasting will be removed in pandas 3.0 "
-        "(at which point this option will be deprecated).",
+        "This option is deprecated and will be removed in a future version. "
+        "It has no effect.",
         validator=is_one_of_factory([True, False]),
     )
+
+# GH#59502
+cf.deprecate_option("future.no_silent_downcasting", Pandas4Warning)

pandas/io/formats/info.py

@@ -1096,7 +1096,7 @@ def headers(self) -> Sequence[str]:
 
     def _gen_rows_without_counts(self) -> Iterator[Sequence[str]]:
         """Iterator with string representation of body data without counts."""
-        yield from self._gen_dtypes()
+        yield from ([dtype] for dtype in self._gen_dtypes())
 
     def _gen_rows_with_counts(self) -> Iterator[Sequence[str]]:
         """Iterator with string representation of body data with counts."""

pandas/tests/arithmetic/test_string.py

@@ -112,3 +112,25 @@ def test_pyarrow_numpy_string_invalid():
 
     with pytest.raises(TypeError, match="Invalid comparison"):
         ser > ser4
+
+
+def test_mul_bool_invalid(any_string_dtype):
+    # GH#62595
+    dtype = any_string_dtype
+    ser = Series(["a", "b", "c"], dtype=dtype)
+
+    if dtype == object:
+        pytest.skip("This is not expect to raise")
+    elif dtype.storage == "python":
+        msg = "Cannot multiply StringArray by bools. Explicitly cast to integers"
+    else:
+        msg = "Can only string multiply by an integer"
+
+    with pytest.raises(TypeError, match=msg):
+        False * ser
+    with pytest.raises(TypeError, match=msg):
+        ser * True
+    with pytest.raises(TypeError, match=msg):
+        ser * np.array([True, False, True], dtype=bool)
+    with pytest.raises(TypeError, match=msg):
+        np.array([True, False, True], dtype=bool) * ser