Merge remote-tracking branch 'upstream/2.3.x' into depr/array-copy-false-futurewarning

Author: jorisvandenbossche

pandas/core/array_algos/replace.py

@@ -149,4 +149,6 @@ def re_replacer(s):
     if mask is None:
         values[:] = f(values)
     else:
+        if values.ndim != mask.ndim:
+            mask = np.broadcast_to(mask, values.shape)
         values[mask] = f(values[mask])

pandas/core/internals/blocks.py

@@ -929,7 +929,7 @@ def replace(
                     blocks = blk.convert(
                         copy=False,
                         using_cow=using_cow,
-                        convert_string=convert_string or self.dtype != _dtype_obj,
+                        convert_string=convert_string or self.dtype == "string",
                     )
                     if len(blocks) > 1 or blocks[0].dtype != blk.dtype:
                         warnings.warn(
@@ -987,7 +987,7 @@ def _replace_regex(
         inplace: bool = False,
         mask=None,
         using_cow: bool = False,
-        convert_string: bool = True,
+        convert_string=None,
         already_warned=None,
     ) -> list[Block]:
         """
@@ -1048,10 +1048,18 @@ def _replace_regex(
                 already_warned.warned_already = True
 
         nbs = block.convert(
-            copy=False, using_cow=using_cow, convert_string=convert_string
+            copy=False,
+            using_cow=using_cow,
+            convert_string=convert_string or self.dtype == "string",
         )
         opt = get_option("future.no_silent_downcasting")
-        if (len(nbs) > 1 or nbs[0].dtype != block.dtype) and not opt:
+        if (
+            len(nbs) > 1
+            or (
+                nbs[0].dtype != block.dtype
+                and not (self.dtype == "string" and nbs[0].dtype == "string")
+            )
+        ) and not opt:
             warnings.warn(
                 # GH#54710
                 "Downcasting behavior in `replace` is deprecated and "
@@ -1088,7 +1096,7 @@ def replace_list(
             values._replace(to_replace=src_list, value=dest_list, inplace=True)
             return [blk]
 
-        convert_string = self.dtype != _dtype_obj
+        convert_string = self.dtype == "string"
 
         # Exclude anything that we know we won't contain
         pairs = [
@@ -2167,6 +2175,13 @@ def where(
                 if isinstance(self.dtype, (IntervalDtype, StringDtype)):
                     # TestSetitemFloatIntervalWithIntIntervalValues
                     blk = self.coerce_to_target_dtype(orig_other)
+                    if (
+                        self.ndim == 2
+                        and isinstance(orig_cond, np.ndarray)
+                        and orig_cond.ndim == 1
+                        and not is_1d_only_ea_dtype(blk.dtype)
+                    ):
+                        orig_cond = orig_cond[:, None]
                     nbs = blk.where(orig_other, orig_cond, using_cow=using_cow)
                     return self._maybe_downcast(
                         nbs, downcast=_downcast, using_cow=using_cow, caller="where"

pandas/io/formats/style.py

@@ -1580,7 +1580,7 @@ def _update_ctx_header(self, attrs: DataFrame, axis: AxisInt) -> None:
         for j in attrs.columns:
             ser = attrs[j]
             for i, c in ser.items():
-                if not c:
+                if not c or pd.isna(c):
                     continue
                 css_list = maybe_convert_css_to_tuples(c)
                 if axis == 0:

pandas/io/pytables.py

@@ -5274,6 +5274,8 @@ def _dtype_to_kind(dtype_str: str) -> str:
         kind = "integer"
     elif dtype_str == "object":
         kind = "object"
+    elif dtype_str == "str":
+        kind = "str"
     else:
         raise ValueError(f"cannot interpret dtype of [{dtype_str}]")
 

pandas/tests/base/test_conversion.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.compat import HAS_PYARROW
 from pandas.compat.numpy import np_version_gt2
 
@@ -391,9 +389,6 @@ def test_to_numpy(arr, expected, zero_copy, index_or_series_or_array):
         assert np.may_share_memory(result_nocopy1, result_nocopy2)
 
 
-@pytest.mark.xfail(
-    using_string_dtype() and not HAS_PYARROW, reason="TODO(infer_string)", strict=False
-)
 @pytest.mark.parametrize("as_series", [True, False])
 @pytest.mark.parametrize(
     "arr", [np.array([1, 2, 3], dtype="int64"), np.array(["a", "b", "c"], dtype=object)]
@@ -405,13 +400,13 @@ def test_to_numpy_copy(arr, as_series, using_infer_string):
 
     # no copy by default
     result = obj.to_numpy()
-    if using_infer_string and arr.dtype == object:
+    if using_infer_string and arr.dtype == object and obj.dtype.storage == "pyarrow":
         assert np.shares_memory(arr, result) is False
     else:
         assert np.shares_memory(arr, result) is True
 
     result = obj.to_numpy(copy=False)
-    if using_infer_string and arr.dtype == object:
+    if using_infer_string and arr.dtype == object and obj.dtype.storage == "pyarrow":
         assert np.shares_memory(arr, result) is False
     else:
         assert np.shares_memory(arr, result) is True

pandas/tests/frame/methods/test_cov_corr.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 import pandas.util._test_decorators as td
 
 import pandas as pd
@@ -328,7 +326,6 @@ def test_corrwith(self, datetime_frame, dtype):
         for row in index[:4]:
             tm.assert_almost_equal(correls[row], df1.loc[row].corr(df2.loc[row]))
 
-    @pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)")
     def test_corrwith_with_objects(self, using_infer_string):
         df1 = DataFrame(
             np.random.default_rng(2).standard_normal((10, 4)),
@@ -342,9 +339,8 @@ def test_corrwith_with_objects(self, using_infer_string):
         df2["obj"] = "bar"
 
         if using_infer_string:
-            import pyarrow as pa
-
-            with pytest.raises(pa.lib.ArrowNotImplementedError, match="has no kernel"):
+            msg = "Cannot perform reduction 'mean' with string dtype"
+            with pytest.raises(TypeError, match=msg):
                 df1.corrwith(df2)
         else:
             with pytest.raises(TypeError, match="Could not convert"):

pandas/tests/frame/methods/test_dtypes.py

@@ -3,8 +3,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
 
 import pandas as pd
@@ -144,13 +142,9 @@ def test_dtypes_timedeltas(self):
         )
         tm.assert_series_equal(result, expected)
 
-    @pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)")
     def test_frame_apply_np_array_return_type(self, using_infer_string):
         # GH 35517
         df = DataFrame([["foo"]])
         result = df.apply(lambda col: np.array("bar"))
-        if using_infer_string:
-            expected = Series([np.array(["bar"])])
-        else:
-            expected = Series(["bar"])
+        expected = Series(np.array("bar"))
         tm.assert_series_equal(result, expected)

pandas/tests/frame/methods/test_fillna.py

@@ -1,8 +1,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 import pandas.util._test_decorators as td
 
 from pandas import (
@@ -91,8 +89,6 @@ def test_fillna_datetime(self, datetime_frame):
         with pytest.raises(ValueError, match=msg):
             datetime_frame.fillna(5, method="ffill")
 
-    # TODO(infer_string) test as actual error instead of xfail
-    @pytest.mark.xfail(using_string_dtype(), reason="can't fill 0 in string")
     def test_fillna_mixed_type(self, float_string_frame):
         mf = float_string_frame
         mf.loc[mf.index[5:20], "foo"] = np.nan
@@ -126,7 +122,7 @@ def test_fillna_empty(self, using_copy_on_write):
                 df.x.fillna(method=m, inplace=True)
                 df.x.fillna(method=m)
 
-    def test_fillna_different_dtype(self, using_infer_string):
+    def test_fillna_different_dtype(self):
         # with different dtype (GH#3386)
         df = DataFrame(
             [["a", "a", np.nan, "a"], ["b", "b", np.nan, "b"], ["c", "c", np.nan, "c"]]
@@ -136,6 +132,7 @@ def test_fillna_different_dtype(self, using_infer_string):
         expected = DataFrame(
             [["a", "a", "foo", "a"], ["b", "b", "foo", "b"], ["c", "c", "foo", "c"]]
         )
+        # column is originally float (all-NaN) -> filling with string gives object dtype
         expected[2] = expected[2].astype("object")
         tm.assert_frame_equal(result, expected)
 
@@ -654,18 +651,10 @@ def test_fillna_col_reordering(self):
             filled = df.fillna(method="ffill")
         assert df.columns.tolist() == filled.columns.tolist()
 
-    # TODO(infer_string) test as actual error instead of xfail
-    @pytest.mark.xfail(using_string_dtype(), reason="can't fill 0 in string")
-    def test_fill_corner(self, float_frame, float_string_frame):
-        mf = float_string_frame
-        mf.loc[mf.index[5:20], "foo"] = np.nan
-        mf.loc[mf.index[-10:], "A"] = np.nan
-
-        filled = float_string_frame.fillna(value=0)
-        assert (filled.loc[filled.index[5:20], "foo"] == 0).all()
-        del float_string_frame["foo"]
-
-        float_frame.reindex(columns=[]).fillna(value=0)
+    def test_fill_empty(self, float_frame):
+        df = float_frame.reindex(columns=[])
+        result = df.fillna(value=0)
+        tm.assert_frame_equal(result, df)
 
     def test_fillna_downcast_dict(self):
         # GH#40809