Add PositionalIndex and EmptyIndex types and align indexes API (#951)

* Add PositionalIndex and EmptyIndex types and align indexes API

Motivation: removing types.None as representation of default pd.RangeIndex
created when index=None is used in DF and Series ctors, that requires
most of Series and DF implementations to branch basing on index types.
Instead all Series and DF functions should use common indexes API
aligned to pandas one.

Changed in this PR:
- types.None index is removed;
- Added EmptyIndexType (instead types.None index) to represent empty
pandas index with dtype='Object' i.e. pd.Index([]);
- Added PositionalIndexType as a replacement for types.None index for
non-empty DFs and Series;
- Changed unboxing of RangeIndex objects to either RangeIndexType of
PositionalIndexType depending on whether it's default range or not;
- Changed fix_df_index and dataframe_constructor.py to create
EmptyIndexType or other index depending on whether tuple of columns is empty;
- Moved implementations for index types from common and
numpy_like functions to specific index files;
- Updated operators and other Series methods implementations to avoid
branching on index types and used index objects API instead;
- Reorganized index tests and added tests verifying specific function
(e.g. series reindexing) for all types of indexes.

* Resolving minor issues and fixing tests

* Fixing types.None index in read_csv

* Fixing PEP remarks

* Revert back change in Series.astype breaking float to str conversion

Author: kozlov-alexey

sdc/datatypes/common_functions.py

@@ -41,12 +41,13 @@
 from numba.core.typing import signature
 from numba import literally
 
-from sdc.datatypes.common_functions import sdc_arrays_argsort, _sdc_asarray, _sdc_take
+from sdc.datatypes.common_functions import sdc_arrays_argsort, _sdc_asarray
 from sdc.datatypes.hpat_pandas_groupby_types import DataFrameGroupByType, SeriesGroupByType
 from sdc.utilities.sdc_typing_utils import TypeChecker, kwsparams2list, sigparams2list
 from sdc.utilities.utils import (sdc_overload, sdc_overload_method, sdc_register_jitable)
 from sdc.hiframes.pd_series_type import SeriesType
 from sdc.str_ext import string_type
+from sdc.functions.numpy_like import take as nplike_take
 
 
 performance_limitation = "This function may reveal slower performance than Pandas* on user system.\
@@ -218,15 +219,15 @@ def _sdc_pandas_groupby_generic_func_codegen(func_name, columns, column_loc,
             f'  column_data_{i} = {df}._data[{type_id}][{col_id}]',
             f'  for j in numpy.arange(res_index_len):',
             f'    idx = argsorted_index[j] if {groupby_param_sort} else j',
-            f'    group_arr_{i} = _sdc_take(column_data_{i}, list({groupby_dict}[group_keys[idx]]))',
+            f'    group_arr_{i} = sdc_take(column_data_{i}, list({groupby_dict}[group_keys[idx]]))',
             f'    group_series_{i} = pandas.Series(group_arr_{i})',
             f'    result_data_{i}[j] = group_series_{i}.{func_name}({extra_impl_params})',
         ]
 
     data = ', '.join(f'\'{column_names[i]}\': result_data_{i}' for i in range(len(columns)))
     func_lines.extend(['\n'.join([
         f'  if {groupby_param_sort}:',
-        f'    res_index = _sdc_take(group_keys, argsorted_index)',
+        f'    res_index = sdc_take(group_keys, argsorted_index)',
         f'  else:',
         f'    res_index = group_keys',
         f'  return pandas.DataFrame({{{data}}}, index=res_index)'
@@ -236,7 +237,7 @@ def _sdc_pandas_groupby_generic_func_codegen(func_name, columns, column_loc,
     global_vars = {'pandas': pandas,
                    'numpy': numpy,
                    '_sdc_asarray': _sdc_asarray,
-                   '_sdc_take': _sdc_take,
+                   'sdc_take': nplike_take,
                    'sdc_arrays_argsort': sdc_arrays_argsort}
 
     return func_text, global_vars
@@ -262,11 +263,11 @@ def _sdc_pandas_series_groupby_generic_func_codegen(func_name, func_params, defa
         f'  result_data = numpy.empty(res_index_len, dtype=res_dtype)',
         f'  for j in numpy.arange(res_index_len):',
         f'    idx = argsorted_index[j] if {groupby_param_sort} else j',
-        f'    group_arr = _sdc_take({series}._data, list({groupby_dict}[group_keys[idx]]))',
+        f'    group_arr = sdc_take({series}._data, list({groupby_dict}[group_keys[idx]]))',
         f'    group_series = pandas.Series(group_arr)',
         f'    result_data[j] = group_series.{func_name}({extra_impl_params})',
         f'  if {groupby_param_sort}:',
-        f'    res_index = _sdc_take(group_keys, argsorted_index)',
+        f'    res_index = sdc_take(group_keys, argsorted_index)',
         f'  else:',
         f'    res_index = group_keys',
         f'  return pandas.Series(data=result_data, index=res_index, name={series}._name)'
@@ -276,7 +277,7 @@ def _sdc_pandas_series_groupby_generic_func_codegen(func_name, func_params, defa
     global_vars = {'pandas': pandas,
                    'numpy': numpy,
                    '_sdc_asarray': _sdc_asarray,
-                   '_sdc_take': _sdc_take,
+                   'sdc_take': nplike_take,
                    'sdc_arrays_argsort': sdc_arrays_argsort}
 
     return func_text, global_vars

sdc/datatypes/hpat_pandas_dataframe_functions.py

@@ -89,7 +89,7 @@ def hpat_pandas_stringmethods_upper_impl(self):
 from sdc.utilities.utils import sdc_overload_method, sdc_register_jitable
 from sdc.hiframes.api import get_nan_mask
 from sdc.str_arr_ext import str_arr_set_na_by_mask, create_str_arr_from_list
-from sdc.datatypes.common_functions import SDCLimitation
+from sdc.utilities.sdc_typing_utils import SDCLimitation
 
 
 @sdc_overload_method(StringMethodsType, 'center')

sdc/datatypes/hpat_pandas_groupby_functions.py

@@ -0,0 +1,32 @@
+# *****************************************************************************
+# Copyright (c) 2020, Intel Corporation All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     Redistributions of source code must retain the above copyright notice,
+#     this list of conditions and the following disclaimer.
+#
+#     Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *****************************************************************************
+
+# modules are dependent on each other e.g. positional_index_type
+# needs range_index_type to be imported, so below order matters
+from .range_index_type import RangeIndexType
+from .positional_index_type import PositionalIndexType
+from .empty_index_type import EmptyIndexType
+from .int64_index_type import Int64IndexType

sdc/datatypes/hpat_pandas_series_functions.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# *****************************************************************************
+# Copyright (c) 2020, Intel Corporation All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     Redistributions of source code must retain the above copyright notice,
+#     this list of conditions and the following disclaimer.
+#
+#     Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *****************************************************************************
+
+from numba import types
+from numba.extending import (
+    models,
+    register_model,
+    make_attribute_wrapper
+)
+
+
+class EmptyIndexType(types.Type):
+
+    # this index represents special case of pd.Index([]) with dtype='object'
+    # for overload typing functions assume it has following dtype
+    dtype = types.pyobject
+
+    def __init__(self, is_named=False):
+        self.is_named = is_named
+        super(EmptyIndexType, self).__init__(
+            name='EmptyIndexType({})'.format(is_named))
+
+
+@register_model(EmptyIndexType)
+class EmptyIndexModel(models.StructModel):
+    def __init__(self, dmm, fe_type):
+
+        name_type = types.unicode_type if fe_type.is_named else types.none
+        members = [
+            ('name', name_type),
+        ]
+        models.StructModel.__init__(self, dmm, fe_type, members)
+
+
+# FIXME_Numba#3372: add into numba.types to allow returning from objmode
+types.EmptyIndexType = EmptyIndexType
+
+
+make_attribute_wrapper(EmptyIndexType, 'name', '_name')

sdc/datatypes/hpat_pandas_stringmethods_functions.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# *****************************************************************************
+# Copyright (c) 2020, Intel Corporation All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     Redistributions of source code must retain the above copyright notice,
+#     this list of conditions and the following disclaimer.
+#
+#     Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *****************************************************************************
+
+from numba import types
+from numba.extending import (
+    models,
+    register_model,
+    make_attribute_wrapper
+)
+
+from sdc.datatypes.indexes import RangeIndexType
+
+
+class PositionalIndexType(types.IterableType):
+    dtype = types.int64
+
+    def __init__(self, is_named=False):
+        self.data = RangeIndexType(is_named)
+        self.is_named = is_named
+        super(PositionalIndexType, self).__init__(
+            name='PositionalIndexType({})'.format(is_named))
+
+    @property
+    def iterator_type(self):
+        res = self.data.iterator_type
+        return res
+
+
+@register_model(PositionalIndexType)
+class PositionalIndexModel(models.StructModel):
+    def __init__(self, dmm, fe_type):
+
+        members = [
+            ('data', fe_type.data),
+        ]
+        models.StructModel.__init__(self, dmm, fe_type, members)
+
+
+# FIXME_Numba#3372: add into numba.types to allow returning from objmode
+types.PositionalIndexType = PositionalIndexType
+
+
+make_attribute_wrapper(PositionalIndexType, 'data', '_data')