[to main branch] Pandas uptake for py3.8 and py3.9+ (#682)

Author: liudmylaru

.github/workflows/run-unittests-default_setup.yml

@@ -3,6 +3,7 @@ name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
 on:
   workflow_dispatch:
   pull_request:
+    branches: [ "main" ]
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -43,13 +44,15 @@ jobs:
         name: "Test config setup"
 
       - name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
-        timeout-minutes: 15
+        timeout-minutes: 30
         shell: bash
         env:
           NoDependency: True
         run: |
           set -x # print commands that are executed
           $CONDA/bin/conda init
           source /home/runner/.bashrc
+          conda install python=${{ matrix.python-version }}
           pip install -r test-requirements.txt
+          conda list
           python -m pytest -v -p no:warnings --durations=5 tests/unitary/default_setup

.github/workflows/run-unittests-py38-cov-report.yml

@@ -3,6 +3,7 @@ name: "[Py3.8][COV REPORT] tests/unitary/**"
 on:
   workflow_dispatch:
   pull_request:
+    branches: [ "main" ]
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -33,6 +34,7 @@ jobs:
             test-path: "tests/unitary"
             # `model` tests running in "slow_tests",
             # `feature_store` tests has its own test suite
+            # 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
@@ -59,13 +61,18 @@ jobs:
 
       - uses: ./.github/workflows/test-env-setup
         name: "Test env setup"
-        timeout-minutes: 20
+        timeout-minutes: 30
 
       # Installing forecast deps for python3.8 test setup only, it will not work with python3.9/3.10, because
       # automlx do not support py3.9 and some versions of py3.10. This step omitted in -py39-py30.yml workflow
       - name: "Install Forecasting dependencies"
         run: |
           pip install -e ".[forecast]"
+      # Installing pii deps for python3.8 test setup only, it will not work with python3.9/3.10, because
+      # 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
+      - name: "Install PII dependencies"
+        run: |
+          pip install -e ".[pii]"
       - name: "Install featurestore marketplace dependencies"
         run: |
           pip install -e ".[feature-store-marketplace]"

.github/workflows/run-unittests-py39-py310.yml

@@ -35,11 +35,13 @@ jobs:
             # `model` tests running in "slow_tests",
             # `feature_store` tests has its own test suite
             # `forecast` tests not supported in python 3.9,3.10 (automlx dependency). Tests are running in python3.8 test env, see run-unittests-py38-cov-report.yml
+            # 'pii' tests run only with py3.8, 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
             # 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
               --ignore tests/unitary/with_extras/operator/forecast \
+              --ignore tests/unitary/with_extras/operator/pii \
               --ignore tests/unitary/with_extras/hpo
           - name: "slow_tests"
             test-path: "tests/unitary/with_extras/model"
@@ -63,7 +65,7 @@ jobs:
 
       - uses: ./.github/workflows/test-env-setup
         name: "Test env setup"
-        timeout-minutes: 20
+        timeout-minutes: 30
 
       - name: "Run hpo tests"
         timeout-minutes: 10

.github/workflows/test-env-setup/action.yml

@@ -14,5 +14,6 @@ runs:
         sudo apt-get install libkrb5-dev graphviz
         $CONDA/bin/conda init
         source /home/runner/.bashrc
-
+        conda install python=${{ matrix.python-version }}
         pip install -r dev-requirements.txt
+        conda list

README-development.md

@@ -101,7 +101,8 @@ To run all unit test install extra dependencies to test all modules of ADS ASD.
 
 ```bash
   # Update your environment with tests dependencies
-  pip install -r dev-requirements.txt
+  pip install -r test-requirements.txt
+  pip install -e ".[testsuite]"
   # Run all unit tests
   python3 -m pytest tests/unitary
 ```
@@ -113,7 +114,8 @@ To run all but opctl integration tests, you can run:
 
 ```bash
   # Update your environment with tests dependencies
-  pip install -r dev-requirements.txt
+  pip install -r test-requirements.txt
+  pip install -e ".[testsuite]"
   # Run integration tests
   python3 -m pytest tests/integration --ignore=tests/integration/opctl
 ```

dev-requirements.txt

@@ -1,18 +1,3 @@
 -r test-requirements.txt
--e ".[bds,data,geo,huggingface,llm,notebook,onnx,opctl,optuna,pii,spark,tensorflow,text,torch,viz]"
-arff
-category_encoders
-cohere
-dask
-faiss-cpu
-fastparquet
-imbalanced-learn
-lxml
-mysql-connector-python
-nltk
-opensearch-py
-pdfplumber
-py4j
-pyarrow
-tables
-xlrd>=1.2.0
+-e ".[bds,data,geo,huggingface,llm,notebook,onnx,opctl,optuna,spark,tensorflow,text,torch,viz]"
+-e ".[testsuite]"

pyproject.toml

@@ -7,7 +7,9 @@
 # PEP 517 – A build-system independent format for source trees - https://peps.python.org/pep-0517/
 # Till recently flit-core library was suggested by pip, so we used it. In future, cosider to change to
 # other, if better, build-backend library.
-requires = ["flit-core >=3.8,<4"]  # should specify <4, so won’t be impacted by changes in the next major version
+requires = [
+  "flit-core >=3.8,<4",
+] # should specify <4, so won’t be impacted by changes in the next major version
 build-backend = "flit_core.buildapi"
 
 
@@ -18,17 +20,15 @@ build-backend = "flit_core.buildapi"
 # PEP 518 – Specifying Minimum Build System Requirements for Python Projects https://peps.python.org/pep-0518/
 
 # Required
-name = "oracle_ads"  # the install (PyPI) name; name for local build in [tool.flit.module] section below
+name = "oracle_ads" # the install (PyPI) name; name for local build in [tool.flit.module] section below
 version = "2.10.1"
 
 # Optional
 description = "Oracle Accelerated Data Science SDK"
-readme = {file = "README.md", content-type = "text/markdown"}
+readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.8"
-license = {file = "LICENSE.txt"}
-authors = [
-  {name = "Oracle Data Science"}
-]
+license = { file = "LICENSE.txt" }
+authors = [{ name = "Oracle Data Science" }]
 keywords = [
   "Oracle Cloud Infrastructure",
   "OCI",
@@ -54,7 +54,7 @@ classifiers = [
 # In dependencies se "<library>; platform_machine == 'aarch64'" to specify ARM underlying platform
 # Copied from install_requires list in setup.py, setup.py got removed in favor of this config file
 dependencies = [
-  "PyYAML>=6",  # pyyaml 5.4 is broken with cython 3
+  "PyYAML>=6",                         # pyyaml 5.4 is broken with cython 3
   "asteval>=0.9.25",
   "cerberus>=1.3.4",
   "cloudpickle>=1.6.0",
@@ -65,7 +65,8 @@ dependencies = [
   "numpy>=1.19.2",
   "oci>=2.113.0",
   "ocifs>=1.1.3",
-  "pandas>1.2.1,<2.1",
+  "pandas>1.2.1; python_version<'3.9'", # starting pandas v2.1.0 requires-python = '>=3.9'
+  "pandas>=2.2.0; python_version>='3.9'",
   "psutil>=5.7.2",
   "python_jsonschema_objects>=0.3.13",
   "requests",
@@ -76,13 +77,9 @@ dependencies = [
 
 [project.optional-dependencies]
 # Copied from extras_require list in setup.py, setup.py got removed in favor of this config file
-bds = [
-  "hdfs[kerberos]",
-  "ibis-framework[impala]",
-  "sqlalchemy",
-]
+bds = ["hdfs[kerberos]", "ibis-framework[impala]", "sqlalchemy"]
 boosted = [
-  "lightgbm<4.0.0",  # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
+  "lightgbm<4.0.0", # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
   "xgboost",
 ]
 data = [
@@ -94,22 +91,14 @@ data = [
   "pandavro>=1.6.0",
   "sqlalchemy>=1.4.1, <=1.4.46",
 ]
-geo = [
-  "geopandas",
-  "oracle_ads[viz]",
-]
-huggingface = [
-  "transformers",
-]
-notebook = [
-  "ipython>=7.23.1, <8.0",
-  "ipywidgets~=7.6.3",
-]
+geo = ["geopandas", "oracle_ads[viz]"]
+huggingface = ["transformers"]
+notebook = ["ipython>=7.23.1, <8.0", "ipywidgets~=7.6.3"]
 onnx = [
-  "lightgbm<4.0.0",  # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
+  "lightgbm<4.0.0",            # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
   "onnx>=1.12.0",
   "onnxmltools>=1.10.0",
-  "onnxruntime>=1.10.0,<1.16",  # v1.16 introduced issues https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
+  "onnxruntime>=1.10.0,<1.16", # v1.16 introduced issues https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
   "oracle_ads[viz]",
   "protobuf<=3.20",
   "skl2onnx>=1.10.4",
@@ -126,26 +115,11 @@ opctl = [
   "py-cpuinfo",
   "rich",
 ]
-optuna = [
-  "optuna==2.9.0",
-  "oracle_ads[viz]",
-]
-spark = [
-  "pyspark>=3.0.0",
-]
-tensorflow = [
-  "oracle_ads[viz]",
-  "tensorflow",
-]
-text = [
-  "spacy",
-  "wordcloud>=1.8.1",
-]
-torch = [
-  "oracle_ads[viz]",
-  "torch",
-  "torchvision",
-]
+optuna = ["optuna==2.9.0", "oracle_ads[viz]"]
+spark = ["pyspark>=3.0.0"]
+tensorflow = ["oracle_ads[viz]", "tensorflow"]
+text = ["spacy", "wordcloud>=1.8.1"]
+torch = ["oracle_ads[viz]", "torch", "torchvision"]
 viz = [
   "bokeh>=3.0.0, <3.2.0",  # starting 3.2.0 bokeh not supporting python3.8; relax after ADS will drop py3.8 support
   "folium>=0.12.1",
@@ -203,9 +177,34 @@ pii = [
   "spacy-transformers==1.2.5",
   "spacy==3.6.1",
 ]
-llm = [
-  "langchain>=0.0.295",
-  "evaluate>=0.4.0",
+llm = ["langchain>=0.1.10", "evaluate>=0.4.0"]
+aqua = ["fire", "cachetools"]
+
+# To reduce backtracking (decrese deps install time) during test/dev env setup reducing number of versions pip is
+# trying to use. Ref - https://pip.pypa.io/en/stable/topics/dependency-resolution/#possible-ways-to-reduce-backtracking.
+# Revisit this section continuously and update to recent version of libraries. focus on pyt3.9/3.10 versions.
+testsuite = [
+  "dask==2023.5.0; python_version=='3.8'",
+  "dask==2023.10.1; python_version>='3.9'", # oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1"
+  "arff",
+  "category_encoders==2.6.3",  # set version to avoid backtracking
+  "cohere==4.53",  # set version to avoid backtracking
+  "dask==2023.10.1; python_version>='3.9'", # oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1"
+  "dask==2023.5.0; python_version=='3.8'",
+  "faiss-cpu",
+  "fastparquet==2024.2.0",  # set version to avoid backtracking
+  "imbalanced-learn",
+  "lxml",
+  "mysql-connector-python",
+  "nltk",
+  "opensearch-py",
+  "pdfplumber",
+  "py4j",
+  "pyarrow",
+  "statsmodels; python_version=='3.8'",
+  "statsmodels>=0.14.1; python_version>='3.9'",  # cython3.0 compatibility added in v0.14.1
+  "tables",
+  "xlrd>=1.2.0",
 ]
 
 
@@ -217,7 +216,7 @@ llm = [
 ads = "ads.cli:cli"
 
 [tool.flit.module]
-name = "ads"  # name for local build and import, see https://flit.pypa.io/en/latest/pyproject_toml.html#module-section
+name = "ads" # name for local build and import, see https://flit.pypa.io/en/latest/pyproject_toml.html#module-section
 
 [tool.flit.sdist]
 # By default `ads` folder and `LICENSE.txt` file included in sdist. Folders `docs` and `tests` are excluded, as weel as other project files

test-requirements.txt

@@ -3,6 +3,7 @@ click
 coverage
 faker
 mock
+parameterized
 pip
 pytest
 pytest-cov