Backport PR #57329 on branch 2.2.x (REGR: CategoricalIndex.difference with null values) (#57336)

Backport PR #57329: REGR: CategoricalIndex.difference with null values

Co-authored-by: Luke Manley <lukemanley@gmail.com>

Author: meeseeksmachine

doc/source/whatsnew/v2.2.1.rst

@@ -20,6 +20,7 @@ Fixed regressions
 - Fixed regression in :func:`wide_to_long` raising an ``AttributeError`` for string columns (:issue:`57066`)
 - Fixed regression in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.SeriesGroupBy.idxmax` ignoring the ``skipna`` argument (:issue:`57040`)
 - Fixed regression in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.SeriesGroupBy.idxmax` where values containing the minimum or maximum value for the dtype could produce incorrect results (:issue:`57040`)
+- Fixed regression in :meth:`CategoricalIndex.difference` raising ``KeyError`` when other contains null values other than NaN (:issue:`57318`)
 - Fixed regression in :meth:`DataFrame.loc` raising ``IndexError`` for non-unique, masked dtype indexes where result has more than 10,000 rows (:issue:`57027`)
 - Fixed regression in :meth:`DataFrame.sort_index` not producing a stable sort for a index with duplicates (:issue:`57151`)
 - Fixed regression in :meth:`DataFrame.to_dict` with ``orient='list'`` and datetime or timedelta types returning integers (:issue:`54824`)

pandas/core/indexes/base.py

@@ -3663,9 +3663,12 @@ def difference(self, other, sort=None):
 
     def _difference(self, other, sort):
         # overridden by RangeIndex
+        this = self
+        if isinstance(self, ABCCategoricalIndex) and self.hasnans and other.hasnans:
+            this = this.dropna()
         other = other.unique()
-        the_diff = self[other.get_indexer_for(self) == -1]
-        the_diff = the_diff if self.is_unique else the_diff.unique()
+        the_diff = this[other.get_indexer_for(this) == -1]
+        the_diff = the_diff if this.is_unique else the_diff.unique()
         the_diff = _maybe_try_sort(the_diff, sort)
         return the_diff
 

pandas/tests/indexes/categorical/test_setops.py

@@ -0,0 +1,18 @@
+import numpy as np
+import pytest
+
+from pandas import (
+    CategoricalIndex,
+    Index,
+)
+import pandas._testing as tm
+
+
+@pytest.mark.parametrize("na_value", [None, np.nan])
+def test_difference_with_na(na_value):
+    # GH 57318
+    ci = CategoricalIndex(["a", "b", "c", None])
+    other = Index(["c", na_value])
+    result = ci.difference(other)
+    expected = CategoricalIndex(["a", "b"], categories=["a", "b", "c"])
+    tm.assert_index_equal(result, expected)