simplify fill_null calls for string[pyarrow] case

Author: jorisvandenbossche

pandas/core/arrays/_arrow_string_mixins.py

@@ -9,18 +9,13 @@
 
 import numpy as np
 
-from pandas._libs import (
-    lib,
-    missing as libmissing,
-)
+from pandas._libs import lib
 from pandas.compat import (
     pa_version_under10p1,
     pa_version_under13p0,
     pa_version_under17p0,
 )
 
-from pandas.core.dtypes.missing import isna
-
 if not pa_version_under10p1:
     import pyarrow as pa
     import pyarrow.compute as pc
@@ -36,12 +31,9 @@
         Self,
     )
 
-    from pandas.core.dtypes.dtypes import ExtensionDtype
-
 
 class ArrowStringArrayMixin:
     _pa_array: Sized
-    dtype: ExtensionDtype
 
     def __init__(self, *args, **kwargs) -> None:
         raise NotImplementedError
@@ -151,12 +143,6 @@ def _str_startswith(
 
                 for p in pat[1:]:
                     result = pc.or_(result, pc.starts_with(self._pa_array, pattern=p))
-        if (
-            self.dtype.na_value is libmissing.NA
-            and na is not lib.no_default
-            and not isna(na)
-        ):  # pyright: ignore [reportGeneralTypeIssues]
-            result = result.fill_null(na)
         return self._convert_bool_result(result, na=na, method_name="startswith")
 
     def _str_endswith(
@@ -174,12 +160,6 @@ def _str_endswith(
 
                 for p in pat[1:]:
                     result = pc.or_(result, pc.ends_with(self._pa_array, pattern=p))
-        if (
-            self.dtype.na_value is libmissing.NA
-            and na is not lib.no_default
-            and not isna(na)
-        ):  # pyright: ignore [reportGeneralTypeIssues]
-            result = result.fill_null(na)
         return self._convert_bool_result(result, na=na, method_name="endswith")
 
     def _str_isalnum(self):
@@ -234,12 +214,6 @@ def _str_contains(
         else:
             pa_contains = pc.match_substring
         result = pa_contains(self._pa_array, pat, ignore_case=not case)
-        if (
-            self.dtype.na_value is libmissing.NA
-            and na is not lib.no_default
-            and not isna(na)
-        ):  # pyright: ignore [reportGeneralTypeIssues]
-            result = result.fill_null(na)
         return self._convert_bool_result(result, na=na, method_name="contains")
 
     def _str_find(self, sub: str, start: int = 0, end: int | None = None):

pandas/core/arrays/arrow/array.py

@@ -2312,6 +2312,8 @@ def _apply_elementwise(self, func: Callable) -> list[list[Any]]:
         ]
 
     def _convert_bool_result(self, result, na=lib.no_default, method_name=None):
+        if na is not lib.no_default and not isna(na):  # pyright: ignore [reportGeneralTypeIssues]
+            result = result.fill_null(na)
         return type(self)(result)
 
     def _convert_int_result(self, result):

pandas/core/arrays/string_arrow.py

@@ -224,26 +224,26 @@ def insert(self, loc: int, item) -> ArrowStringArray:
         return super().insert(loc, item)
 
     def _convert_bool_result(self, values, na=lib.no_default, method_name=None):
+        if na is not lib.no_default and not isna(na) and not isinstance(na, bool):
+            # GH#59561
+            warnings.warn(
+                f"Allowing a non-bool 'na' in obj.str.{method_name} is deprecated "
+                "and will raise in a future version.",
+                FutureWarning,
+                stacklevel=find_stack_level(),
+            )
+            na = bool(na)
+
         if self.dtype.na_value is np.nan:
-            na_value: bool | lib.NoDefault
-            if na is lib.no_default:
-                na_value = False
-            elif isna(na):
+            if na is lib.no_default or isna(na):
                 # NaN propagates as False
                 values = values.fill_null(False)
-                na_value = lib.no_default
             else:
-                if not isinstance(na, bool):
-                    # GH#59561
-                    warnings.warn(
-                        f"Allowing a non-bool 'na' in obj.str.{method_name} is "
-                        "deprecated and will raise in a future version.",
-                        FutureWarning,
-                        stacklevel=find_stack_level(),
-                    )
-                values = values.fill_null(bool(na))
-                na_value = lib.no_default
-            return ArrowExtensionArray(values).to_numpy(na_value=na_value)
+                values = values.fill_null(na)
+            return values.to_numpy(zero_copy_only=False)
+        else:
+            if na is not lib.no_default and not isna(na):  # pyright: ignore [reportGeneralTypeIssues]
+                values = values.fill_null(na)
         return BooleanDtype().__from_arrow__(values)
 
     def _maybe_convert_setitem_value(self, value):
@@ -325,21 +325,6 @@ def _str_contains(
                 fallback_performancewarning()
             return super()._str_contains(pat, case, flags, na, regex)
 
-        if (
-            self.dtype.na_value is libmissing.NA
-            and na is not lib.no_default
-            and not isna(na)
-        ):
-            if not isinstance(na, bool):
-                # GH#59561
-                warnings.warn(
-                    "Allowing a non-bool 'na' in obj.str.contains is deprecated "
-                    "and will raise in a future version.",
-                    FutureWarning,
-                    stacklevel=find_stack_level(),
-                )
-                na = bool(na)
-
         return ArrowStringArrayMixin._str_contains(self, pat, case, flags, na, regex)
 
     def _str_replace(

pandas/tests/strings/test_find_replace.py

@@ -298,10 +298,7 @@ def test_startswith_endswith_validate_na(any_string_dtype):
     )
 
     dtype = ser.dtype
-    if (
-        isinstance(dtype, pd.StringDtype)
-        and (dtype.storage == "python" or dtype.na_value is np.nan)
-    ) or dtype == np.dtype("object"):
+    if (isinstance(dtype, pd.StringDtype)) or dtype == np.dtype("object"):
         msg = "Allowing a non-bool 'na' in obj.str.startswith is deprecated"
         with tm.assert_produces_warning(FutureWarning, match=msg):
             ser.str.startswith("kapow", na="baz")