BUG: Fix MultiIndex construction in pd.concat() with Int64Dtype NA

Author: parthava-adabala

doc/source/whatsnew/v3.0.0.rst

@@ -1140,6 +1140,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc` with inconsistent behavior of loc-set with 2 given indexes to Series (:issue:`59933`)
 - Bug in :meth:`Index.equals` when comparing between :class:`Series` with string dtype :class:`Index` (:issue:`61099`)
 - Bug in :meth:`Index.get_indexer` and similar methods when ``NaN`` is located at or after position 128 (:issue:`58924`)
+- Bug in :func:`pandas.concat` incorrectly constructing the :class:`MultiIndex` when an inner level contained :obj:`pandas.NA` with :class:`pandas.Int64Dtype`, causing a :exc:`KeyError` on lookup (:issue:`62903`)
 - Bug in :meth:`MultiIndex.insert` when a new value inserted to a datetime-like level gets cast to ``NaT`` and fails indexing (:issue:`60388`)
 - Bug in :meth:`Series.__setitem__` when assigning boolean series with boolean indexer will raise ``LossySetitemError`` (:issue:`57338`)
 - Bug in printing :attr:`Index.names` and :attr:`MultiIndex.levels` would not escape single quotes (:issue:`60190`)

pandas/core/reshape/concat.py

@@ -990,8 +990,9 @@ def _make_concat_multiindex(indexes, keys, levels=None, names=None) -> MultiInde
         new_levels.extend(new_index.levels)
         new_codes.extend(np.tile(lab, kpieces) for lab in new_index.codes)
     else:
-        new_levels.append(new_index.unique())
-        single_codes = new_index.unique().get_indexer(new_index)
+        levels_for_index = new_index.unique().dropna()
+        new_levels.append(levels_for_index)
+        single_codes = levels_for_index.get_indexer(new_index)
         new_codes.append(np.tile(single_codes, kpieces))
 
     if len(new_names) < len(new_levels):

pandas/tests/indexes/multi/test_constructors.py

@@ -343,6 +343,24 @@ def test_from_arrays_respects_none_names():
 
     tm.assert_index_equal(result, expected)
 
+def test_concat_int64dtype_na_multiindex_lookup():
+    levels1 = ['a', 'b']
+    levels2 = pd.Series([1, 2, pd.NA], dtype=pd.Int64Dtype())
+    index1 = MultiIndex.from_product([levels1, levels2], names=['one', 'two'])
+    series1 = pd.Series([f'{i1}-{i2}' for i1, i2 in index1], index=index1)
+    series2 = pd.concat(
+        [series1.loc[i1] for i1 in levels1], 
+        keys=levels1, 
+        names=['one']
+    )
+    lookup_key = ('a', pd.NA) 
+    result = series2.at[lookup_key]
+    assert result == 'a-<NA>'
+    level_two = series2.index.levels[1]
+    codes_two = series2.index.codes[1]
+    assert level_two.hasnans is False
+    assert codes_two[2] == -1
+    assert codes_two[5] == -1
 
 # ----------------------------------------------------------------------------
 # from_tuples