Introduce config options which enables parallel/inline for overloads (#457)

* Introduce config options which enables parallel/inline for overloads

* Implement strtobool via distutils.util.strtobool

Author: AlexanderKalistratov

sdc/config.py

@@ -47,7 +47,13 @@
     _has_opencv = True
     import sdc.cv_ext
 
-config_transport_mpi_default = distutils_util.strtobool(os.getenv('SDC_CONFIG_MPI', 'True'))
+
+def strtobool(val):
+    '''Convert string to True or False'''
+    return bool(distutils_util.strtobool(val))
+
+
+config_transport_mpi_default = strtobool(os.getenv('SDC_CONFIG_MPI', 'True'))
 '''
 Default value for transport used if no function decorator controls the transport
 '''
@@ -58,11 +64,21 @@
 because decorator called later then modules have been initialized
 '''
 
-config_pipeline_hpat_default = distutils_util.strtobool(os.getenv('SDC_CONFIG_PIPELINE_SDC', 'False'))
+config_pipeline_hpat_default = strtobool(os.getenv('SDC_CONFIG_PIPELINE_SDC', 'False'))
 '''
 Default value used to select compiler pipeline in a function decorator
 '''
 
+config_use_parallel_overloads = strtobool(os.getenv('SDC_AUTO_PARALLEL', 'False'))
+'''
+Default value used to select whether auto parallel would be applied to sdc functions
+'''
+
+config_inline_overloads = strtobool(os.getenv('SDC_AUTO_INLINE', 'False'))
+'''
+Default value used to select whether sdc functions would inline
+'''
+
 if not config_pipeline_hpat_default:
     # avoid using MPI transport if no SDC compiler pipeline used
     config_transport_mpi_default = False
@@ -73,7 +89,7 @@
 Default value for a pointer intended to use as Numba.DefaultPassBuilder.define_nopython_pipeline() in overloaded function
 '''
 
-test_expected_failure  = distutils_util.strtobool(os.getenv('SDC_TEST_EXPECTED_FAILURE', 'False'))
+test_expected_failure = strtobool(os.getenv('SDC_TEST_EXPECTED_FAILURE', 'False'))
 '''
 If True then replaces skip decorators to expectedFailure decorator.
 '''

sdc/tests/test_series.py

@@ -37,7 +37,7 @@
 from sdc.tests.test_base import TestCase
 from sdc.tests.test_utils import (
     count_array_REPs, count_parfor_REPs, count_array_OneDs, get_start_end,
-    skip_numba_jit, skip_sdc_jit, sdc_limitation)
+    skip_numba_jit, skip_sdc_jit, sdc_limitation, skip_parallel, skip_inline)
 from sdc.tests.gen_test_data import ParquetGenerator
 from numba import types
 from numba.config import IS_32BITS
@@ -612,6 +612,7 @@ def test_impl(A, deep):
                             self.assertEqual(actual.index is S.index, expected.index is S.index)
                             self.assertEqual(actual.index is S.index, not deep)
 
+    @skip_parallel
     @skip_sdc_jit('Series.corr() parameter "min_periods" unsupported')
     def test_series_corr(self):
         def test_series_corr_impl(S1, S2, min_periods=None):
@@ -679,6 +680,8 @@ def test_series_corr_impl(S1, S2, min_periods=None):
         msg = 'Method corr(). The object min_periods'
         self.assertIn(msg, str(raises.exception))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_int_to_str1(self):
         '''Verifies Series.astype implementation with function 'str' as argument
            converts integer series to series of strings
@@ -691,6 +694,8 @@ def test_impl(S):
         S = pd.Series(np.arange(n))
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_int_to_str2(self):
         '''Verifies Series.astype implementation with a string literal dtype argument
            converts integer series to series of strings
@@ -703,6 +708,8 @@ def test_impl(S):
         S = pd.Series(np.arange(n))
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_str_to_str1(self):
         '''Verifies Series.astype implementation with function 'str' as argument
            handles string series not changing it
@@ -714,6 +721,8 @@ def test_impl(S):
         S = pd.Series(['aa', 'bb', 'cc'])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_str_to_str2(self):
         '''Verifies Series.astype implementation with a string literal dtype argument
            handles string series not changing it
@@ -725,6 +734,8 @@ def test_impl(S):
         S = pd.Series(['aa', 'bb', 'cc'])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_str_to_str_index_str(self):
         '''Verifies Series.astype implementation with function 'str' as argument
            handles string series not changing it
@@ -738,6 +749,8 @@ def test_impl(S):
         S = pd.Series(['aa', 'bb', 'cc'], index=['d', 'e', 'f'])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_str_to_str_index_int(self):
         '''Verifies Series.astype implementation with function 'str' as argument
            handles string series not changing it
@@ -857,6 +870,8 @@ def test_impl(A):
         S = pd.Series(['3.24', '1E+05', '-1', '-1.3E-01', 'nan', 'inf'])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_str_index_str(self):
         '''Verifies Series.astype implementation with function 'str' as argument
            handles string series not changing it
@@ -869,6 +884,8 @@ def test_impl(S):
         S = pd.Series(['aa', 'bb', 'cc'], index=['a', 'b', 'c'])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     def test_series_astype_str_index_int(self):
         '''Verifies Series.astype implementation with function 'str' as argument
            handles string series not changing it
@@ -1462,6 +1479,7 @@ def test_impl(A):
         df = pd.DataFrame({'A': np.arange(n)})
         self.assertTrue(isinstance(hpat_func(df.A), np.ndarray))
 
+    @skip_parallel
     @skip_sdc_jit('No support of axis argument in old-style Series.fillna() impl')
     def test_series_fillna_axis1(self):
         '''Verifies Series.fillna() implementation handles 'index' as axis argument'''
@@ -1472,6 +1490,7 @@ def test_impl(S):
         S = pd.Series([1.0, 2.0, np.nan, 1.0, np.inf])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
     @skip_sdc_jit('No support of axis argument in old-style Series.fillna() impl')
     def test_series_fillna_axis2(self):
         '''Verifies Series.fillna() implementation handles 0 as axis argument'''
@@ -1482,6 +1501,7 @@ def test_impl(S):
         S = pd.Series([1.0, 2.0, np.nan, 1.0, np.inf])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
     @skip_sdc_jit('No support of axis argument in old-style Series.fillna() impl')
     def test_series_fillna_axis3(self):
         '''Verifies Series.fillna() implementation handles correct non-literal axis argument'''
@@ -1493,6 +1513,7 @@ def test_impl(S, axis):
         for axis in [0, 'index']:
             pd.testing.assert_series_equal(hpat_func(S, axis), test_impl(S, axis))
 
+    @skip_parallel
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_float_from_df(self):
         '''Verifies Series.fillna() applied to a named float Series obtained from a DataFrame'''
@@ -1504,6 +1525,7 @@ def test_impl(S):
         df = pd.DataFrame({'A': [1.0, 2.0, np.nan, 1.0, np.inf]})
         pd.testing.assert_series_equal(hpat_func(df.A), test_impl(df.A), check_names=False)
 
+    @skip_parallel
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_float_index1(self):
         '''Verifies Series.fillna() implementation for float series with default index'''
@@ -1515,6 +1537,7 @@ def test_impl(S):
             S = pd.Series(data)
             pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_float_index2(self):
         '''Verifies Series.fillna() implementation for float series with string index'''
@@ -1525,6 +1548,7 @@ def test_impl(S):
         S = pd.Series([1.0, 2.0, np.nan, 1.0, np.inf], ['a', 'b', 'c', 'd', 'e'])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_float_index3(self):
         def test_impl(S):
@@ -1534,6 +1558,8 @@ def test_impl(S):
         S = pd.Series([1.0, 2.0, np.nan, 1.0, np.inf], index=[1, 2, 5, 7, 10])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_str_from_df(self):
         '''Verifies Series.fillna() applied to a named float Series obtained from a DataFrame'''
@@ -1546,6 +1572,8 @@ def test_impl(S):
         pd.testing.assert_series_equal(hpat_func(df.A),
                                        test_impl(df.A), check_names=False)
 
+    @skip_parallel
+    @skip_inline
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_str_index1(self):
         '''Verifies Series.fillna() implementation for series of strings with default index'''
@@ -1556,6 +1584,8 @@ def test_impl(S):
         S = pd.Series(['aa', 'b', None, 'cccd', ''])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_str_index2(self):
         '''Verifies Series.fillna() implementation for series of strings with string index'''
@@ -1566,6 +1596,8 @@ def test_impl(S):
         S = pd.Series(['aa', 'b', None, 'cccd', ''], ['a', 'b', 'c', 'd', 'e'])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
+    @skip_inline
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_str_index3(self):
         def test_impl(S):
@@ -1576,6 +1608,7 @@ def test_impl(S):
         S = pd.Series(['aa', 'b', None, 'cccd', ''], index=[1, 2, 5, 7, 10])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
     @skip_sdc_jit('BUG: old-style fillna impl returns series without index')
     def test_series_fillna_float_inplace1(self):
         '''Verifies Series.fillna() implementation for float series with default index and inplace argument True'''
@@ -1588,6 +1621,7 @@ def test_impl(S):
         S2 = S1.copy()
         pd.testing.assert_series_equal(hpat_func(S1), test_impl(S2))
 
+    @skip_parallel
     @unittest.skip('TODO: add reflection support and check method return value')
     def test_series_fillna_float_inplace2(self):
         '''Verifies Series.fillna(inplace=True) results are reflected back in the original float series'''
@@ -1601,6 +1635,7 @@ def test_impl(S):
         self.assertIsNone(test_impl(S2))
         pd.testing.assert_series_equal(S1, S2)
 
+    @skip_parallel
     def test_series_fillna_float_inplace3(self):
         '''Verifies Series.fillna() implementation correcly handles omitted inplace argument as default False'''
         def test_impl(S):
@@ -1612,6 +1647,7 @@ def test_impl(S):
         pd.testing.assert_series_equal(hpat_func(S1), test_impl(S1))
         pd.testing.assert_series_equal(S1, S2)
 
+    @skip_parallel
     def test_series_fillna_inplace_non_literal(self):
         '''Verifies Series.fillna() implementation handles only Boolean literals as inplace argument'''
         def test_impl(S, param):
@@ -1623,6 +1659,7 @@ def test_impl(S, param):
         expected = ValueError if sdc.config.config_pipeline_hpat_default else TypingError
         self.assertRaises(expected, hpat_func, S, True)
 
+    @skip_parallel
     @skip_numba_jit('TODO: investigate why Numba types inplace as bool (non-literal value)')
     def test_series_fillna_str_inplace1(self):
         '''Verifies Series.fillna() implementation for series of strings
@@ -1687,6 +1724,7 @@ def test_impl(S):
         S = pd.Series([pd.NaT, pd.Timestamp('1970-12-01'), pd.Timestamp('2012-07-25')])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
     def test_series_fillna_bool_no_index1(self):
         '''Verifies Series.fillna() implementation for bool series with default index'''
         def test_impl(S):
@@ -2570,6 +2608,7 @@ def test_impl(S):
         S = pd.Series([np.nan, -2., 3., 0.5E-01, 0xFF, 0o7, 0b101])
         pd.testing.assert_series_equal(hpat_func(S), test_impl(S))
 
+    @skip_parallel
     def test_series_cov1(self):
         def test_impl(S1, S2):
             return S1.cov(S2)
@@ -4021,6 +4060,7 @@ def test_impl(A, B):
         B = np.random.ranf(n)
         pd.testing.assert_series_equal(hpat_func(A, B), test_impl(A, B))
 
+    @skip_parallel
     def test_series_sort_values_full(self):
         def test_impl(series, ascending, kind):
             return series.sort_values(axis=0, ascending=ascending, inplace=False, kind=kind, na_position='last')
@@ -4068,6 +4108,7 @@ def test_impl(series, ascending, kind):
                         np.testing.assert_array_equal(ref_result.data, jit_result.data)
                         self.assertEqual(ref, jit)
 
+    @skip_parallel
     def test_series_sort_values_full_idx(self):
         def test_impl(series, ascending, kind):
             return series.sort_values(axis=0, ascending=ascending, inplace=False, kind=kind, na_position='last')
@@ -4688,6 +4729,7 @@ def test_series_count_impl(S):
             result = hpat_func(S)
             self.assertEqual(result, result_ref)
 
+    @skip_parallel
     @skip_sdc_jit('Series.cumsum() np.nan as input data unsupported')
     def test_series_cumsum(self):
         def test_impl():
@@ -4698,6 +4740,7 @@ def test_impl():
         cfunc = self.jit(pyfunc)
         pd.testing.assert_series_equal(pyfunc(), cfunc())
 
+    @skip_parallel
     @skip_sdc_jit('Series.cumsum() np.nan as input data unsupported')
     def test_series_cumsum_unboxing(self):
         def test_impl(s):
@@ -4710,6 +4753,7 @@ def test_impl(s):
             series = pd.Series(data)
             pd.testing.assert_series_equal(pyfunc(series), cfunc(series))
 
+    @skip_parallel
     @skip_sdc_jit('Series.cumsum() parameters "axis", "skipna" unsupported')
     def test_series_cumsum_full(self):
         def test_impl(s, axis, skipna):
@@ -4751,6 +4795,7 @@ def test_impl(s, axis):
             msg = 'Method cumsum(). Unsupported parameters. Given axis: int'
             self.assertIn(msg, str(raises.exception))
 
+    @skip_parallel
     @skip_sdc_jit('Series.cov() parameter "min_periods" unsupported')
     def test_series_cov(self):
         def test_series_cov_impl(S1, S2, min_periods=None):
@@ -5146,6 +5191,7 @@ def test_impl(A, B):
                 B = pd.Series(np.arange(n)**2, index=np.arange(n, dtype=dtype_right))
                 pd.testing.assert_series_equal(hpat_func(A, B), test_impl(A, B), check_dtype=False)
 
+    @skip_parallel
     @skip_sdc_jit('Arithmetic operations on Series with non-default indexes are not supported in old-style')
     def test_series_operator_add_numeric_same_index_numeric_fixme(self):
         """ Same as test_series_operator_add_same_index_numeric but with w/a for the problem.
@@ -5167,6 +5213,7 @@ def test_impl(A, B):
                 B = pd.Series(np.arange(n)**2, index=np.arange(n, dtype=dtype_right))
                 pd.testing.assert_series_equal(hpat_func(A, B), test_impl(A, B), check_dtype=False)
 
+    @skip_parallel
     @skip_sdc_jit('Arithmetic operations on Series with non-default indexes are not supported in old-style')
     def test_series_operator_add_numeric_same_index_str(self):
         """Verifies implementation of Series.operator.add between two numeric Series with the same string indexes"""
@@ -5179,6 +5226,7 @@ def test_impl(A, B):
         B = pd.Series(np.arange(n)**2, index=['a', 'c', 'e', 'c', 'b', 'a', 'o'])
         pd.testing.assert_series_equal(hpat_func(A, B), test_impl(A, B), check_dtype=False, check_names=False)
 
+    @skip_parallel
     @skip_sdc_jit('Arithmetic operations on Series with non-default indexes are not supported in old-style')
     def test_series_operator_add_numeric_align_index_int(self):
         """Verifies implementation of Series.operator.add between two numeric Series with non-equal integer indexes"""
@@ -5195,6 +5243,7 @@ def test_impl(A, B):
         B = pd.Series(np.arange(n)**2, index=index_B)
         pd.testing.assert_series_equal(hpat_func(A, B), test_impl(A, B), check_dtype=False, check_names=False)
 
+    @skip_parallel
     @skip_sdc_jit('Arithmetic operations on Series with non-default indexes are not supported in old-style')
     def test_series_operator_add_numeric_align_index_str(self):
         """Verifies implementation of Series.operator.add between two numeric Series with non-equal string indexes"""
@@ -5228,6 +5277,7 @@ def test_impl(A, B):
         B = pd.Series(np.arange(n)**2, index=index_B)
         pd.testing.assert_series_equal(hpat_func(A, B), test_impl(A, B), check_dtype=False, check_names=False)
 
+    @skip_parallel
     @skip_sdc_jit('Arithmetic operations on Series with non-default indexes are not supported in old-style')
     def test_series_operator_add_numeric_align_index_other_dtype(self):
         """Verifies implementation of Series.operator.add between two numeric Series
@@ -5253,6 +5303,7 @@ def test_impl(A, B):
         B = pd.Series(np.arange(size_B)**2)
         pd.testing.assert_series_equal(hpat_func(A, B), test_impl(A, B), check_dtype=False, check_names=False)
 
+    @skip_parallel
     @skip_sdc_jit('Arithmetic operations on Series requiring alignment of indexes are not supported in old-style')
     def test_series_operator_add_align_index_int_capacity(self):
         """Verifies implementation of Series.operator.add and alignment of numeric indexes of large size"""