Merge branch 'main' into bug/62915-dst-union-fix

Author: antznette1

pandas/_libs/internals.pyi

@@ -94,3 +94,7 @@ class BlockValuesRefs:
     def add_reference(self, blk: Block) -> None: ...
     def add_index_reference(self, index: Index) -> None: ...
     def has_reference(self) -> bool: ...
+
+class SetitemMixin:
+    def __setitem__(self, key, value) -> None: ...
+    def __delitem__(self, key) -> None: ...

pandas/_libs/internals.pyx

@@ -1,6 +1,9 @@
 from collections import defaultdict
+import sys
+import warnings
 
 cimport cython
+from cpython cimport PY_VERSION_HEX
 from cpython.object cimport PyObject
 from cpython.pyport cimport PY_SSIZE_T_MAX
 from cpython.slice cimport PySlice_GetIndicesEx
@@ -20,6 +23,9 @@ from numpy cimport (
 cnp.import_array()
 
 from pandas._libs.algos import ensure_int64
+from pandas.compat import CHAINED_WARNING_DISABLED
+from pandas.errors import ChainedAssignmentError
+from pandas.errors.cow import _chained_assignment_msg
 
 from pandas._libs.util cimport (
     is_array,
@@ -996,3 +1002,43 @@ cdef class BlockValuesRefs:
                 return self._has_reference_maybe_locked()
         ELSE:
             return self._has_reference_maybe_locked()
+
+
+cdef extern from "Python.h":
+    """
+    #if PY_VERSION_HEX < 0x030E0000
+    int __Pyx_PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *ref);
+    #else
+    #define __Pyx_PyUnstable_Object_IsUniqueReferencedTemporary \
+        PyUnstable_Object_IsUniqueReferencedTemporary
+    #endif
+    """
+    int PyUnstable_Object_IsUniqueReferencedTemporary\
+        "__Pyx_PyUnstable_Object_IsUniqueReferencedTemporary"(object o) except -1
+
+
+# Python version compatibility for PyUnstable_Object_IsUniqueReferencedTemporary
+cdef inline bint _is_unique_referenced_temporary(object obj) except -1:
+    if PY_VERSION_HEX >= 0x030E0000:
+        # Python 3.14+ has PyUnstable_Object_IsUniqueReferencedTemporary
+        return PyUnstable_Object_IsUniqueReferencedTemporary(obj)
+    else:
+        # Fallback for older Python versions using sys.getrefcount
+        return sys.getrefcount(obj) <= 1
+
+
+cdef class SetitemMixin:
+    # class used in DataFrame and Series for checking for chained assignment
+
+    def __setitem__(self, key, value) -> None:
+        cdef bint is_unique = 0
+        if not CHAINED_WARNING_DISABLED:
+            is_unique = _is_unique_referenced_temporary(self)
+            if is_unique:
+                warnings.warn(
+                    _chained_assignment_msg, ChainedAssignmentError, stacklevel=1
+                )
+        self._setitem(key, value)
+
+    def __delitem__(self, key) -> None:
+        self._delitem(key)

pandas/_testing/contexts.py

@@ -13,8 +13,8 @@
 import uuid
 
 from pandas.compat import (
-    PYPY,
-    WARNING_CHECK_DISABLED,
+    CHAINED_WARNING_DISABLED,
+    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
 )
 from pandas.errors import ChainedAssignmentError
 
@@ -163,10 +163,18 @@ def with_csv_dialect(name: str, **kwargs) -> Generator[None]:
         csv.unregister_dialect(name)
 
 
-def raises_chained_assignment_error(extra_warnings=(), extra_match=()):
+def raises_chained_assignment_error(
+    extra_warnings=(), extra_match=(), inplace_method=False
+):
     from pandas._testing import assert_produces_warning
 
-    if PYPY or WARNING_CHECK_DISABLED:
+    WARNING_DISABLED = (
+        CHAINED_WARNING_DISABLED_INPLACE_METHOD
+        if inplace_method
+        else CHAINED_WARNING_DISABLED
+    )
+
+    if WARNING_DISABLED:
         if not extra_warnings:
             from contextlib import nullcontext
 

pandas/compat/__init__.py

@@ -16,12 +16,13 @@
 from typing import TYPE_CHECKING
 
 from pandas.compat._constants import (
+    CHAINED_WARNING_DISABLED,
+    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
     IS64,
     ISMUSL,
     PY312,
     PY314,
     PYPY,
-    WARNING_CHECK_DISABLED,
     WASM,
 )
 from pandas.compat.numpy import is_numpy_dev
@@ -152,14 +153,15 @@ def is_ci_environment() -> bool:
 
 
 __all__ = [
+    "CHAINED_WARNING_DISABLED",
+    "CHAINED_WARNING_DISABLED_INPLACE_METHOD",
     "HAS_PYARROW",
     "IS64",
     "ISMUSL",
     "PY312",
     "PY314",
     "PYARROW_MIN_VERSION",
     "PYPY",
-    "WARNING_CHECK_DISABLED",
     "WASM",
     "is_numpy_dev",
     "pa_version_under14p0",

pandas/compat/_constants.py

@@ -19,7 +19,8 @@
 WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"])
 ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "")
 REF_COUNT = 2
-WARNING_CHECK_DISABLED = PY314
+CHAINED_WARNING_DISABLED = PYPY or (PY314 and not sys._is_gil_enabled())  # type: ignore[attr-defined]
+CHAINED_WARNING_DISABLED_INPLACE_METHOD = PYPY or PY314
 
 
 __all__ = [

pandas/compat/pickle_compat.py

@@ -22,6 +22,7 @@
     PeriodArray,
     TimedeltaArray,
 )
+from pandas.core.generic import NDFrame
 from pandas.core.internals import BlockManager
 
 if TYPE_CHECKING:
@@ -90,6 +91,10 @@ def load_reduce(self) -> None:
                 cls = args[0]
                 stack[-1] = NDArrayBacked.__new__(*args)
                 return
+            elif args and issubclass(args[0], NDFrame):
+                cls = args[0]
+                stack[-1] = cls.__new__(cls)
+                return
             raise
 
     dispatch[pickle.REDUCE[0]] = load_reduce  # type: ignore[assignment]

pandas/core/frame.py

@@ -48,11 +48,11 @@
     properties,
 )
 from pandas._libs.hashtable import duplicated
+from pandas._libs.internals import SetitemMixin
 from pandas._libs.lib import is_range_indexer
-from pandas.compat import PYPY
 from pandas.compat._constants import (
+    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
     REF_COUNT,
-    WARNING_CHECK_DISABLED,
 )
 from pandas.compat._optional import import_optional_dependency
 from pandas.compat.numpy import function as nv
@@ -63,7 +63,6 @@
 )
 from pandas.errors.cow import (
     _chained_assignment_method_msg,
-    _chained_assignment_msg,
 )
 from pandas.util._decorators import (
     Appender,
@@ -521,7 +520,7 @@
 
 
 @set_module("pandas")
-class DataFrame(NDFrame, OpsMixin):
+class DataFrame(SetitemMixin, NDFrame, OpsMixin):
     """
     Two-dimensional, size-mutable, potentially heterogeneous tabular data.
 
@@ -668,6 +667,11 @@ class DataFrame(NDFrame, OpsMixin):
     #  and ExtensionArray.  Should NOT be overridden by subclasses.
     __pandas_priority__ = 4000
 
+    # override those to avoid inheriting from SetitemMixin (cython generates
+    # them by default)
+    __reduce__ = object.__reduce__
+    __setstate__ = NDFrame.__setstate__
+
     @property
     def _constructor(self) -> type[DataFrame]:
         return DataFrame
@@ -4321,7 +4325,8 @@ def isetitem(self, loc, value) -> None:
         arraylike, refs = self._sanitize_column(value)
         self._iset_item_mgr(loc, arraylike, inplace=False, refs=refs)
 
-    def __setitem__(self, key, value) -> None:
+    # def __setitem__() is implemented in SetitemMixin and dispatches to this method
+    def _setitem(self, key, value) -> None:
         """
         Set item(s) in DataFrame by key.
 
@@ -4405,12 +4410,6 @@ def __setitem__(self, key, value) -> None:
         z  3  50
         # Values for 'a' and 'b' are completely ignored!
         """
-        if not PYPY and not WARNING_CHECK_DISABLED:
-            if sys.getrefcount(self) <= REF_COUNT + 1:
-                warnings.warn(
-                    _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
-                )
-
         key = com.apply_if_callable(key, self)
 
         # see if we can slice the rows
@@ -9363,7 +9362,7 @@ def update(
         1  2  500.0
         2  3    6.0
         """
-        if not PYPY and not WARNING_CHECK_DISABLED:
+        if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
             if sys.getrefcount(self) <= REF_COUNT:
                 warnings.warn(
                     _chained_assignment_method_msg,

pandas/core/generic.py

@@ -82,10 +82,9 @@
     WriteExcelBuffer,
     npt,
 )
-from pandas.compat import PYPY
 from pandas.compat._constants import (
+    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
     REF_COUNT,
-    WARNING_CHECK_DISABLED,
 )
 from pandas.compat._optional import import_optional_dependency
 from pandas.compat.numpy import function as nv
@@ -2075,7 +2074,6 @@ def __getstate__(self) -> dict[str, Any]:
             **meta,
         }
 
-    @final
     def __setstate__(self, state) -> None:
         if isinstance(state, BlockManager):
             self._mgr = state
@@ -4261,8 +4259,9 @@ def _slice(self, slobj: slice, axis: AxisInt = 0) -> Self:
         result = result.__finalize__(self)
         return result
 
+    # def __delitem__() is implemented in SetitemMixin and dispatches to this method
     @final
-    def __delitem__(self, key) -> None:
+    def _delitem(self, key) -> None:
         """
         Delete item
         """
@@ -7077,7 +7076,7 @@ def fillna(
         """
         inplace = validate_bool_kwarg(inplace, "inplace")
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,
@@ -7324,7 +7323,7 @@ def ffill(
         """
         inplace = validate_bool_kwarg(inplace, "inplace")
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,
@@ -7464,7 +7463,7 @@ def bfill(
         """
         inplace = validate_bool_kwarg(inplace, "inplace")
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,
@@ -7549,7 +7548,7 @@ def replace(
 
         inplace = validate_bool_kwarg(inplace, "inplace")
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,
@@ -7912,7 +7911,7 @@ def interpolate(
         inplace = validate_bool_kwarg(inplace, "inplace")
 
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,
@@ -8567,7 +8566,7 @@ def clip(
         inplace = validate_bool_kwarg(inplace, "inplace")
 
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,
@@ -10202,7 +10201,7 @@ def where(
         """
         inplace = validate_bool_kwarg(inplace, "inplace")
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,
@@ -10266,7 +10265,7 @@ def mask(
     ) -> Self | None:
         inplace = validate_bool_kwarg(inplace, "inplace")
         if inplace:
-            if not PYPY and not WARNING_CHECK_DISABLED:
+            if not CHAINED_WARNING_DISABLED_INPLACE_METHOD:
                 if sys.getrefcount(self) <= REF_COUNT:
                     warnings.warn(
                         _chained_assignment_method_msg,

pandas/core/indexing.py

@@ -15,10 +15,9 @@
 
 from pandas._libs.indexing import NDFrameIndexerBase
 from pandas._libs.lib import item_from_zerodim
-from pandas.compat import PYPY
 from pandas.compat._constants import (
+    CHAINED_WARNING_DISABLED,
     REF_COUNT,
-    WARNING_CHECK_DISABLED,
 )
 from pandas.errors import (
     AbstractMethodError,
@@ -920,7 +919,7 @@ def _ensure_listlike_indexer(self, key, axis=None, value=None) -> None:
 
     @final
     def __setitem__(self, key, value) -> None:
-        if not PYPY and not WARNING_CHECK_DISABLED:
+        if not CHAINED_WARNING_DISABLED:
             if sys.getrefcount(self.obj) <= REF_COUNT:
                 warnings.warn(
                     _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
@@ -2588,7 +2587,7 @@ def __getitem__(self, key):
         return super().__getitem__(key)
 
     def __setitem__(self, key, value) -> None:
-        if not PYPY and not WARNING_CHECK_DISABLED:
+        if not CHAINED_WARNING_DISABLED:
             if sys.getrefcount(self.obj) <= REF_COUNT:
                 warnings.warn(
                     _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
@@ -2619,7 +2618,7 @@ def _convert_key(self, key):
         return key
 
     def __setitem__(self, key, value) -> None:
-        if not PYPY and not WARNING_CHECK_DISABLED:
+        if not CHAINED_WARNING_DISABLED:
             if sys.getrefcount(self.obj) <= REF_COUNT:
                 warnings.warn(
                     _chained_assignment_msg, ChainedAssignmentError, stacklevel=2