diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 27d5a65a08467..bc4220c17d862 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1076,6 +1076,7 @@ The affected cases are: df + arr[[0], :] .. ipython:: python + :okwarning: # Comparison operations and arithmetic operations both broadcast. df == (1, 2) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 0396d1704b579..369e415c13594 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -803,6 +803,7 @@ Other Deprecations - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.unstack` and :meth:`DataFrame.unstack` (:issue:`12189`, :issue:`53868`) - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.shift` and :meth:`DataFrame.shift` (:issue:`53802`) - Deprecated allowing strings representing full dates in :meth:`DataFrame.at_time` and :meth:`Series.at_time` (:issue:`50839`) +- Deprecated arithmetic operations between pandas objects (:class:`DataFrame`, :class:`Series`, :class:`Index`, and pandas-implemented :class:`ExtensionArray` subclasses) and list-likes other than ``list``, ``np.ndarray``, :class:`ExtensionArray`, :class:`Index`, :class:`Series`, :class:`DataFrame`. For e.g. ``tuple`` or ``range``, explicitly cast these to a supported object instead. In a future version, these will be treated as scalar-like for pointwise operation (:issue:`62423`) - Deprecated backward-compatibility behavior for :meth:`DataFrame.select_dtypes` matching "str" dtype when ``np.object_`` is specified (:issue:`61916`) - Deprecated option "future.no_silent_downcasting", as it is no longer used. In a future version accessing this option will raise (:issue:`59502`) - Deprecated passing non-Index types to :meth:`Index.join`; explicitly convert to Index first (:issue:`62897`) diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py index 53c938faf9257..5b51c8469f61d 100644 --- a/pandas/core/arrays/arrow/array.py +++ b/pandas/core/arrays/arrow/array.py @@ -957,6 +957,20 @@ def _op_method_error_message(self, other, op) -> str: ) def _evaluate_op_method(self, other, op, arrow_funcs) -> Self: + if ( + is_list_like(other) + and not isinstance(other, (np.ndarray, ExtensionArray, list)) + and not ops.has_castable_attr(other) + ): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) + pa_type = self._pa_array.type other_original = other other = self._box_pa(other) diff --git a/pandas/core/arrays/boolean.py b/pandas/core/arrays/boolean.py index 2d7bae7833f29..106ad27027102 100644 --- a/pandas/core/arrays/boolean.py +++ b/pandas/core/arrays/boolean.py @@ -7,6 +7,7 @@ Self, cast, ) +import warnings import numpy as np @@ -14,7 +15,9 @@ lib, missing as libmissing, ) +from pandas.errors import Pandas4Warning from pandas.util._decorators import set_module +from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.common import is_list_like from pandas.core.dtypes.dtypes import register_extension_dtype @@ -22,6 +25,7 @@ from pandas.core import ops from pandas.core.array_algos import masked_accumulations +from pandas.core.arrays import ExtensionArray from pandas.core.arrays.masked import ( BaseMaskedArray, BaseMaskedDtype, @@ -397,6 +401,18 @@ def _logical_method(self, other, op): if isinstance(other, BooleanArray): other, mask = other._data, other._mask elif is_list_like(other): + if not isinstance( + other, (list, ExtensionArray, np.ndarray) + ) and not ops.has_castable_attr(other): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) + other = np.asarray(other, dtype="bool") if other.ndim > 1: return NotImplemented diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 1fbcd0665c467..504cbdf8511ed 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -78,6 +78,7 @@ from pandas.errors import ( AbstractMethodError, InvalidComparison, + Pandas4Warning, PerformanceWarning, ) from pandas.util._decorators import ( @@ -976,6 +977,19 @@ def _cmp_method(self, other, op): # TODO: handle 2D-like listlikes return op(self.ravel(), other.ravel()).reshape(self.shape) + if is_list_like(other): + if not isinstance( + other, (list, np.ndarray, ExtensionArray) + ) and not ops.has_castable_attr(other): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) + try: other = self._validate_comparison_value(other) except InvalidComparison: diff --git a/pandas/core/arrays/interval.py b/pandas/core/arrays/interval.py index 3e724b176b76d..818b579029f92 100644 --- a/pandas/core/arrays/interval.py +++ b/pandas/core/arrays/interval.py @@ -13,6 +13,7 @@ TypeAlias, overload, ) +import warnings import numpy as np @@ -38,7 +39,11 @@ npt, ) from pandas.compat.numpy import function as nv -from pandas.errors import IntCastingNaNError +from pandas.errors import ( + IntCastingNaNError, + Pandas4Warning, +) +from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.cast import ( LossySetitemError, @@ -70,6 +75,7 @@ notna, ) +from pandas.core import ops from pandas.core.algorithms import ( isin, take, @@ -854,6 +860,17 @@ def __setitem__(self, key, value) -> None: def _cmp_method(self, other, op): # ensure pandas array for list-like and eliminate non-interval scalars if is_list_like(other): + if not isinstance( + other, (list, np.ndarray, ExtensionArray) + ) and not ops.has_castable_attr(other): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) if len(self) != len(other): raise ValueError("Lengths must match to compare") other = pd_array(other) diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py index 6085b577f4392..11e3c61210b76 100644 --- a/pandas/core/arrays/masked.py +++ b/pandas/core/arrays/masked.py @@ -24,7 +24,11 @@ IS64, is_platform_windows, ) -from pandas.errors import AbstractMethodError +from pandas.errors import ( + AbstractMethodError, + Pandas4Warning, +) +from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.base import ExtensionDtype from pandas.core.dtypes.cast import maybe_downcast_to_dtype @@ -810,6 +814,20 @@ def _arith_method(self, other, op): op_name = op.__name__ omask = None + if ( + is_list_like(other) + and not isinstance(other, (list, np.ndarray, ExtensionArray)) + and not ops.has_castable_attr(other) + ): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) + if ( not hasattr(other, "dtype") and is_list_like(other) @@ -922,6 +940,17 @@ def _cmp_method(self, other, op) -> BooleanArray: other, mask = other._data, other._mask elif is_list_like(other): + if not isinstance( + other, (list, np.ndarray, ExtensionArray) + ) and not ops.has_castable_attr(other): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) other = np.asarray(other) if other.ndim > 1: raise NotImplementedError("can only perform ops with 1-d structures") diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index 83e16f5d4b8db..d0bb7904f5cdf 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -30,7 +30,10 @@ ) from pandas._libs.tslibs import NaT from pandas.compat.numpy import function as nv -from pandas.errors import PerformanceWarning +from pandas.errors import ( + Pandas4Warning, + PerformanceWarning, +) from pandas.util._decorators import doc from pandas.util._exceptions import find_stack_level from pandas.util._validators import ( @@ -66,7 +69,10 @@ notna, ) -from pandas.core import arraylike +from pandas.core import ( + arraylike, + ops, +) import pandas.core.algorithms as algos from pandas.core.arraylike import OpsMixin from pandas.core.arrays import ExtensionArray @@ -1788,6 +1794,18 @@ def _arith_method(self, other, op): return _wrap_result(op_name, result, self.sp_index, fill) else: + if not isinstance( + other, (list, np.ndarray, ExtensionArray) + ) and not ops.has_castable_attr(other): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) + other = np.asarray(other) with np.errstate(all="ignore"): if len(self) != len(other): @@ -1800,6 +1818,19 @@ def _arith_method(self, other, op): return _sparse_array_op(self, other, op, op_name) def _cmp_method(self, other, op) -> SparseArray: + if ( + is_list_like(other) + and not isinstance(other, (list, np.ndarray, ExtensionArray)) + and not ops.has_castable_attr(other) + ): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) if not is_scalar(other) and not isinstance(other, type(self)): # convert list-like to ndarray other = np.asarray(other) diff --git a/pandas/core/arrays/string_.py b/pandas/core/arrays/string_.py index fd68cc513125d..a54cb84cb8bd7 100644 --- a/pandas/core/arrays/string_.py +++ b/pandas/core/arrays/string_.py @@ -1084,6 +1084,17 @@ def _cmp_method(self, other, op): valid = ~mask if lib.is_list_like(other): + if not isinstance( + other, (list, ExtensionArray, np.ndarray) + ) and not ops.has_castable_attr(other): + warnings.warn( + f"Operation with {type(other).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) if len(other) != len(self): # prevent improper broadcasting when other is 2D raise ValueError( diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 0f36ea031d3d4..4368322b74b9f 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -8559,6 +8559,18 @@ def to_series(right): ) elif is_list_like(right) and not isinstance(right, (Series, DataFrame)): + if not isinstance( + right, (np.ndarray, ExtensionArray, Index, list, dict) + ) and not ops.has_castable_attr(right): + warnings.warn( + f"Operation with {type(right).__name__} are deprecated. " + "In a future version these will be treated as scalar-like. " + "To retain the old behavior, explicitly wrap in a Series " + "instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) + # GH#36702. Raise when attempting arithmetic with list of array-like. if any(is_array_like(el) for el in right): raise ValueError( diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index 9f9d69a182f72..a8f9d71a4f1a5 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -17,6 +17,7 @@ ) from pandas.core.ops.common import ( get_op_result_name, + has_castable_attr, unpack_zerodim_and_defer, ) from pandas.core.ops.docstrings import make_flex_doc @@ -71,6 +72,7 @@ "fill_binop", "get_array_op", "get_op_result_name", + "has_castable_attr", "invalid_comparison", "kleene_and", "kleene_or", diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index e0aa4f44fe2be..4855aec7c0415 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -21,6 +21,11 @@ from pandas._typing import F +def has_castable_attr(obj) -> bool: + attrs = ["__array__", "__dlpack__", "__arrow_c_array__", "__arrow_c_stream__"] + return any(hasattr(obj, name) for name in attrs) + + def unpack_zerodim_and_defer(name: str) -> Callable[[F], F]: """ Boilerplate for pandas conventions in arithmetic and comparison methods. diff --git a/pandas/core/series.py b/pandas/core/series.py index c75973e2897f1..c5dbfb0b2cff8 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -6743,6 +6743,15 @@ def _flex_method(self, other, op, *, level=None, fill_value=None, axis: Axis = 0 if isinstance(other, Series): return self._binop(other, op, level=level, fill_value=fill_value) elif isinstance(other, (np.ndarray, list, tuple, ExtensionArray)): + if isinstance(other, tuple): + op_name = op.__name__.strip("_") + warnings.warn( + f"Series.{op_name} with a tuple is deprecated and will be " + "treated as scalar-like in a future version. " + "Explicitly wrap in a numpy array instead.", + Pandas4Warning, + stacklevel=find_stack_level(), + ) if len(other) != len(self): raise ValueError("Lengths must be equal") other = self._constructor(other, self.index, copy=False) diff --git a/pandas/tests/arithmetic/test_interval.py b/pandas/tests/arithmetic/test_interval.py index 0e316cf419cb0..41b9d7a5e134d 100644 --- a/pandas/tests/arithmetic/test_interval.py +++ b/pandas/tests/arithmetic/test_interval.py @@ -207,19 +207,19 @@ def test_compare_list_like_interval_mixed_closed( @pytest.mark.parametrize( "other", [ - ( + [ Interval(0, 1), Interval(Timedelta("1 day"), Timedelta("2 days")), Interval(4, 5, "both"), Interval(10, 20, "neither"), - ), - (0, 1.5, Timestamp("20170103"), np.nan), - ( + ], + [0, 1.5, Timestamp("20170103"), np.nan], + [ Timestamp("20170102", tz="US/Eastern"), Timedelta("2 days"), "baz", pd.NaT, - ), + ], ], ) def test_compare_list_like_object(self, op, interval_array, other): diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index 5a7cad77a9de0..019c6a0362288 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -306,10 +306,16 @@ def test_cmp_dt64_arraylike_tznaive(self, comparison_op): tuple(right), right.astype(object), ]: - result = op(arr, other) + depr_msg = "Operation with tuple are deprecated." + warn = None + if isinstance(other, tuple): + warn = Pandas4Warning + with tm.assert_produces_warning(warn, match=depr_msg): + result = op(arr, other) tm.assert_numpy_array_equal(result, expected) - result = op(other, arr) + with tm.assert_produces_warning(warn, match=depr_msg): + result = op(other, arr) tm.assert_numpy_array_equal(result, expected) diff --git a/pandas/tests/extension/test_numpy.py b/pandas/tests/extension/test_numpy.py index 691ce9341b788..4c7c763a369d4 100644 --- a/pandas/tests/extension/test_numpy.py +++ b/pandas/tests/extension/test_numpy.py @@ -291,6 +291,7 @@ def test_arith_series_with_array(self, data, all_arithmetic_operators): self.series_array_exc = series_array_exc super().test_arith_series_with_array(data, all_arithmetic_operators) + @skip_nested def test_arith_frame_with_scalar(self, data, all_arithmetic_operators, request): opname = all_arithmetic_operators frame_scalar_exc = None diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index 5e50759d34014..fdb3a71dd9147 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -12,6 +12,7 @@ import pytest from pandas.compat._optional import import_optional_dependency +from pandas.errors import Pandas4Warning import pandas as pd from pandas import ( @@ -273,7 +274,9 @@ def test_df_boolean_comparison_error(self): expected = DataFrame([[False, False], [True, False], [False, False]]) - result = df == (2, 2) + depr_msg = "In a future version these will be treated as scalar-like" + with tm.assert_produces_warning(Pandas4Warning, match=depr_msg): + result = df == (2, 2) tm.assert_frame_equal(result, expected) result = df == [2, 2] @@ -1009,7 +1012,14 @@ def test_arith_alignment_non_pandas_object(self, values): # GH#17901 df = DataFrame({"A": [1, 1], "B": [1, 1]}) expected = DataFrame({"A": [2, 2], "B": [3, 3]}) - result = df + values + + depr_msg = "In a future version these will be treated as scalar-like" + warn = None + if isinstance(values, (range, deque, tuple)): + warn = Pandas4Warning + + with tm.assert_produces_warning(warn, match=depr_msg): + result = df + values tm.assert_frame_equal(result, expected) def test_arith_non_pandas_object(self): @@ -1639,9 +1649,11 @@ def test_boolean_comparison(self): # wrong shape df > lst + depr_msg = "In a future version these will be treated as scalar-like" with pytest.raises(ValueError, match=msg1d): # wrong shape - df > tup + with tm.assert_produces_warning(Pandas4Warning, match=depr_msg): + df > tup # broadcasts like ndarray (GH#23000) result = df > b_r @@ -1665,7 +1677,8 @@ def test_boolean_comparison(self): df == lst with pytest.raises(ValueError, match=msg1d): - df == tup + with tm.assert_produces_warning(Pandas4Warning, match=depr_msg): + df == tup # broadcasts like ndarray (GH#23000) result = df == b_r @@ -1690,7 +1703,8 @@ def test_boolean_comparison(self): df == lst with pytest.raises(ValueError, match=msg1d): - df == tup + with tm.assert_produces_warning(Pandas4Warning, match=depr_msg): + df == tup def test_inplace_ops_alignment(self): # inplace ops / ops alignment @@ -1859,13 +1873,22 @@ def test_alignment_non_pandas(self, val): align = DataFrame._align_for_op + depr_msg = "In a future version these will be treated as scalar-like" + warn = None + if isinstance(val, (tuple, range)): + warn = Pandas4Warning + + with tm.assert_produces_warning(warn, match=depr_msg): + result = align(df, val, axis=0)[1] expected = DataFrame({"X": val, "Y": val, "Z": val}, index=df.index) - tm.assert_frame_equal(align(df, val, axis=0)[1], expected) + tm.assert_frame_equal(result, expected) expected = DataFrame( {"X": [1, 1, 1], "Y": [2, 2, 2], "Z": [3, 3, 3]}, index=df.index ) - tm.assert_frame_equal(align(df, val, axis=1)[1], expected) + with tm.assert_produces_warning(warn, match=depr_msg): + result = align(df, val, axis=1)[1] + tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("val", [[1, 2], (1, 2), np.array([1, 2]), range(1, 3)]) def test_alignment_non_pandas_length_mismatch(self, val): @@ -1877,14 +1900,21 @@ def test_alignment_non_pandas_length_mismatch(self, val): columns=columns, ) + depr_msg = "In a future version these will be treated as scalar-like" + warn = None + if isinstance(val, (tuple, range)): + warn = Pandas4Warning + align = DataFrame._align_for_op # length mismatch msg = "Unable to coerce to Series, length must be 3: given 2" with pytest.raises(ValueError, match=msg): - align(df, val, axis=0) + with tm.assert_produces_warning(warn, match=depr_msg): + align(df, val, axis=0) with pytest.raises(ValueError, match=msg): - align(df, val, axis=1) + with tm.assert_produces_warning(warn, match=depr_msg): + align(df, val, axis=1) def test_alignment_non_pandas_index_columns(self): index = ["A", "B", "C"] diff --git a/pandas/tests/series/test_arithmetic.py b/pandas/tests/series/test_arithmetic.py index a9ed61e2c40cb..edfda810c7649 100644 --- a/pandas/tests/series/test_arithmetic.py +++ b/pandas/tests/series/test_arithmetic.py @@ -15,6 +15,7 @@ from pandas._libs import lib from pandas.compat._optional import import_optional_dependency +from pandas.errors import Pandas4Warning import pandas as pd from pandas import ( @@ -916,7 +917,13 @@ def test_series_ops_name_retention(self, flex, box, names, all_binary_operators) if is_logical: # Series doesn't have these as flex methods return - result = getattr(left, name)(right) + + warn = None + tuple_msg = "with a tuple is deprecated" + if box is tuple: + warn = Pandas4Warning + with tm.assert_produces_warning(warn, match=tuple_msg): + result = getattr(left, name)(right) else: if is_logical and box in [list, tuple]: with pytest.raises(TypeError, match=msg):