Merge pull request #978 from IntelPython/master

Merge from master 2021_w22

Author: kozlov-alexey

.bandit

@@ -0,0 +1,2 @@
+[bandit]
+exclude: ./buildscripts/*

README.rst

@@ -37,13 +37,13 @@ Distribution includes Intel® SDC for Python 3.6 and Python 3.7 for Windows and
 
 Intel® SDC conda package can be installed using the steps below::
 
-    > conda create -n sdc-env python=<3.7 or 3.6> pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge
+    > conda create -n sdc-env python=<3.7 or 3.6> -c anaconda -c conda-forge
     > conda activate sdc-env
     > conda install sdc -c intel/label/beta -c intel -c defaults -c conda-forge --override-channels
 
 Intel® SDC wheel package can be installed using the steps below::
 
-    > conda create -n sdc-env python=<3.7 or 3.6> pip pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge
+    > conda create -n sdc-env python=<3.7 or 3.6> pip -c anaconda -c conda-forge
     > conda activate sdc-env
     > pip install --index-url https://pypi.anaconda.org/intel/label/beta/simple --extra-index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple sdc
 

buildscripts/azure/template-linux-macos.yml

@@ -43,3 +43,9 @@ jobs:
         python buildscripts/build_doc.py --python=$PYTHON_VER --sdc-channel=./sdc-build
       displayName: 'Build Intel SDC documentation'
       continueOnError: ${{ parameters.allowFailure }}
+
+    - script: |
+        source $HOME/miniconda3/bin/activate
+        python buildscripts/test_conda_install.py --python=$PYTHON_VER --channels="-c anaconda -c conda-forge"
+      displayName: 'Test SDC installation from conda'
+      continueOnError: ${{ parameters.allowFailure }}

buildscripts/azure/template-windows.yml

@@ -36,3 +36,9 @@ jobs:
         python buildscripts\\run_examples.py --python=%PYTHON_VER% --sdc-channel=.\\sdc-build
       displayName: 'Run Intel SDC examples'
       continueOnError: ${{ parameters.allowFailure }}
+
+    - script: |
+        call C:\\Miniconda\\Scripts\activate.bat
+        python buildscripts\\test_conda_install.py --python=%PYTHON_VER% --channels="-c anaconda -c conda-forge"
+      displayName: 'Test SDC installation from conda'
+      continueOnError: ${{ parameters.allowFailure }}

buildscripts/build_doc.py

@@ -88,12 +88,13 @@ def publish_doc(sdc_utils):
     parser = argparse.ArgumentParser()
     parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
                         help='Python version, default = 3.7')
+    parser.add_argument('--channels', default=None, help='Default env channels')
     parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
     parser.add_argument('--publish', action='store_true', help='Publish documentation to sdc-doc')
 
     args = parser.parse_args()
 
-    sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
+    sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
     sdc_utils.log_info('Build Intel(R) SDC documentation', separate=True)
     sdc_utils.log_info(sdc_utils.line_double)
     sdc_utils.create_environment(['sphinx', 'sphinxcontrib-programoutput'])

buildscripts/run_benchmarks.py

@@ -47,14 +47,15 @@ def run_benchmarks(sdc_utils, args_list, num_threads_list):
     parser = argparse.ArgumentParser()
     parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
                         help='Python version, default = 3.7')
+    parser.add_argument('--channels', default=None, help='Default env channels')
     parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
     parser.add_argument('--args-list', required=True, nargs='+', help='List of arguments sets for benchmarks')
     parser.add_argument('--num-threads-list', required=True, nargs='+',
                         help='List of values for NUMBA_NUM_THREADS env variable')
 
     args = parser.parse_args()
 
-    sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
+    sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
     sdc_utils.log_info('Run Intel(R) SDC benchmarks', separate=True)
     sdc_utils.log_info(sdc_utils.line_double)
     sdc_utils.create_environment(['openpyxl', 'xlrd'])

buildscripts/run_examples.py

@@ -119,11 +119,12 @@ def run_examples(sdc_utils):
     parser = argparse.ArgumentParser()
     parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
                         help='Python version, default = 3.7')
+    parser.add_argument('--channels', default=None, help='Default env channels')
     parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
 
     args = parser.parse_args()
 
-    sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
+    sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
     sdc_utils.log_info('Run Intel(R) SDC examples', separate=True)
     sdc_utils.log_info(sdc_utils.line_double)
     sdc_utils.create_environment()

buildscripts/test.py

@@ -48,11 +48,12 @@ def run_tests(sdc_utils):
     parser = argparse.ArgumentParser()
     parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
                         help='Python version, default = 3.7')
-    parser.add_argument('--sdc-channel', required=True, help='Local Intel SDC channel')
+    parser.add_argument('--channels', default=None, help='Default env channels')
+    parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
 
     args = parser.parse_args()
 
-    sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
+    sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
     sdc_utils.log_info('Run Intel(R) SDC tests', separate=True)
     sdc_utils.log_info(sdc_utils.line_double)
     sdc_utils.create_environment(['conda-build'])

buildscripts/test_conda_install.py

@@ -0,0 +1,63 @@
+# *****************************************************************************
+# Copyright (c) 2019-2021, Intel Corporation All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     Redistributions of source code must retain the above copyright notice,
+#     this list of conditions and the following disclaimer.
+#
+#     Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# *****************************************************************************
+
+
+import argparse
+import os
+import shutil
+import traceback
+import re
+
+from pathlib import Path
+from utilities import SDC_Build_Utilities
+
+
+def check_sdc_installed(sdc_utils, sdc_package):
+    cmd_output = sdc_utils.get_command_output('conda list sdc')
+    pattern = sdc_package.replace('=', r'\s+')
+    return re.search(pattern, cmd_output)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
+                        help='Python version, default = 3.7')
+    parser.add_argument('--channels', default=None, help='Default env channels')
+    parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
+
+    args = parser.parse_args()
+
+    sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel)
+    sdc_utils.log_info('Test Intel(R) SDC conda install', separate=True)
+    sdc_utils.log_info(sdc_utils.line_double)
+    sdc_utils.create_environment()
+    sdc_package = f'sdc={sdc_utils.get_sdc_version_from_channel()}'
+
+    # channels list is aligned with install instruction in README.rst
+    install_channels = "-c intel/label/beta -c intel -c defaults -c conda-forge"
+    sdc_utils.install_conda_package([sdc_package], channels=install_channels)
+
+    assert check_sdc_installed(sdc_utils, sdc_package), "SDC package was not installed"

buildscripts/utilities.py

@@ -25,7 +25,6 @@
 # *****************************************************************************
 
 
-
 import json
 import os
 import platform
@@ -41,7 +40,7 @@
 
 
 class SDC_Build_Utilities:
-    def __init__(self, python, sdc_local_channel=None):
+    def __init__(self, python, channels=None, sdc_channel=None):
         self.src_path = Path(__file__).resolve().parent.parent
         self.env_name = 'sdc_env'
         self.python = python
@@ -52,10 +51,14 @@ def __init__(self, python, sdc_local_channel=None):
         self.line_single = '-'*80
 
         # Set channels
-        self.channel_list = ['-c', 'defaults', '-c', 'conda-forge']
-        if sdc_local_channel:
-            sdc_local_channel = Path(sdc_local_channel).resolve().as_uri()
-            self.channel_list = ['-c', sdc_local_channel] + self.channel_list
+        build_channels = ['-c', 'defaults', '-c', 'conda-forge']
+        self.channel_list = build_channels if channels is None else channels.split()
+        if sdc_channel:
+            self.sdc_channel = Path(sdc_channel).resolve().as_uri()
+            self.channel_list = ['-c', self.sdc_channel] + self.channel_list
+        else:
+            self.sdc_channel = 'intel/label/beta'
+            # keep SDC channel but do not add it to env channels
         self.channels = ' '.join(self.channel_list)
 
         # Conda activate command and conda croot (build) folder
@@ -92,12 +95,13 @@ def create_environment(self, packages_list=[]):
 
         return
 
-    def install_conda_package(self, packages_list):
+    def install_conda_package(self, packages_list, channels=None):
         assert type(packages_list) == list, 'Argument should be a list'
 
         self.log_info(f'Install {" ".join(packages_list)} to {self.env_name} conda environment')
         install_args = ['-n', self.env_name]
-        install_args += self.channel_list + ['--override-channels', '-q', '-y'] + packages_list
+        replace_channels = channels.split() if channels else self.channel_list
+        install_args += replace_channels + ['--override-channels', '-q', '-y'] + packages_list
         self.log_info(self.__run_conda_command(Conda_Commands.INSTALL, install_args))
 
         return
@@ -140,12 +144,11 @@ def log_info(self, msg, separate=False):
     def get_sdc_version_from_channel(self):
         python_version = 'py' + self.python.replace('.', '')
 
-        # Get Intel SDC version from first channel in channel_list
-        search_args = ['sdc', '-c', self.channel_list[1], '--override-channels', '--json']
+        search_args = ['sdc', '-c', self.sdc_channel, '--override-channels', '--json']
         search_result = self.__run_conda_command(Conda_Commands.SEARCH, search_args)
 
         repo_data = json.loads(search_result)
-        for package_data in repo_data['sdc']:
+        for package_data in reversed(repo_data['sdc']):
             sdc_version = package_data['version']
             sdc_build = package_data['build']
             if python_version in sdc_build: