Merge branch 'develop' into release-notes/2.8.4

Author: qiuosier

.github/workflows/publish-to-readthedocs.yml

@@ -0,0 +1,40 @@
+name: "Publish Docs"
+
+on:
+  # Auto-trigger this workflow on tag creation
+  push:
+    tags:
+      - 'v*.*.*'
+  # Auto-trigger this workflow on merge to main changes in docs/** folder
+  pull_request_target:
+    types:
+      - closed
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+
+env:
+  RTDS_ADS_PROJECT: https://readthedocs.org/api/v3/projects/accelerated-data-science
+  RTDS_ADS_TOKEN: ${{ secrets.RTDS_ADS_TOKEN }}
+
+jobs:
+  build-n-publish:
+    name: Build and publish Docs 📖 to Readthedocs
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: When PR ✅ merged - Trigger Readthedocs build
+        if: github.event_name == 'pull_request_target' && github.event.pull_request.merged == true
+        run: |
+          curl \
+            -X POST \
+            -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/latest/builds/
+      - name: When tag 🏷️ pushed - Trigger Readthedocs build
+        if: github.event_name == 'push' && startsWith(github.ref_name, 'v')
+        run: |
+          # add 10 minutes wait time for readthedocs see freshly created tag
+          sleep 10m
+          curl \
+            -X POST \
+            -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/${{ github.ref_name }}/builds/

.github/workflows/run-unittests-default_setup.yml

@@ -1,18 +1,17 @@
 name: tests/unitary/default_setup/**
 
 on:
-  push:
+  pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
       - develop
     paths:
-      - 'ads/**'
+      - "ads/**"
       - setup.py
-      - '**requirements.txt'
-      - '!docs/**'
-
-  pull_request:
+      - "**requirements.txt"
+      - .github/workflows/run-unittests.yml
+      - .github/workflows/run-unittests-default_setup.yml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -77,7 +76,6 @@ jobs:
         timeout-minutes: 15
         shell: bash
         env:
-          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
           NoDependency: True
         run: |
           set -x # print commands that are executed

.github/workflows/run-unittests.yml

@@ -1,18 +1,17 @@
 name: tests/unitary/**
 
 on:
-  push:
+  pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
       - develop
     paths:
-      - 'ads/**'
+      - "ads/**"
       - setup.py
-      - '**requirements.txt'
-      - '!docs/**'
-
-  pull_request:
+      - "**requirements.txt"
+      - .github/workflows/run-unittests.yml
+      - .github/workflows/run-unittests-default_setup.yml
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -88,22 +87,21 @@ jobs:
         shell: bash
         run: |
           set -x # print commands that are executed
-          
+
           sudo apt-get install libkrb5-dev graphviz
           $CONDA/bin/conda init
           source /home/runner/.bashrc
-          
+
           pip install -r dev-requirements.txt
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 60
         shell: bash
         env:
-          NB_SESSION_COMPARTMENT_OCID: ocid1.compartment.oc1.<unique_ocid>
           CONDA_PREFIX: /usr/share/miniconda
         run: |
           set -x # print commands that are executed
-          
+
           # Setup project and tests folder for cov reports to not be overwritten by another parallel step
           if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
             mkdir -p cov-${{ matrix.name }}
@@ -112,7 +110,7 @@ jobs:
             ln -s ../ads ads
             ln -s ../.coveragerc .coveragerc
           fi
-          
+
           # Run tests
           python -m pytest -v -p no:warnings --durations=5 \
           -n auto --dist loadfile ${{ matrix.cov-reports }} \
@@ -148,41 +146,41 @@ jobs:
       - name: "Calculate overall coverage"
         run: |
           set -x # print commands that are executed
-          
+
           # Prepare default cov body text 
           COV_BODY_INTRO="📌 Overall coverage:\n\n"
           echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
-          
+
           # Combine coverage files
           pip install coverage
           coverage combine cov-reports-unitary/.coverage cov-reports-model/.coverage
-          
+
           # Make html report
           coverage html
-          
+
           # Calculate overall coverage and update body message
           COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
           if [[ ! -z $COV ]]; then 
-            if [[ $COV < 50 ]]; then COLOR=red; elif [[ $COV < 80 ]]; then COLOR=yellow; else COLOR=green; fi
+            if [[ $COV -lt 50 ]]; then COLOR=red; elif [[ $COV -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
             echo COV_BODY="$COV_BODY_INTRO ![Coverage-$COV%](https://img.shields.io/badge/coverage-$COV%25-$COLOR)" >> $GITHUB_ENV
           fi
 
       - name: "Calculate coverage diff"
         if: always()
         run: |
           set -x # print commands that are executed
-          
+
           # Prepare default diff body text 
           DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
-          echo DIFF_BODY="$BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
-          
+          echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
+
           # Prepare file paths to coverage xml files
           # Filenames taken from job.test last step with name - "Save coverage files"
           FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1="" 
           FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
           echo "FILE1=$FILE1" >> $GITHUB_ENV
           echo "FILE2=$FILE2" >> $GITHUB_ENV
-          
+
           # Calculate coverage diff and update body message
           pip install diff_cover
           diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }} 
@@ -193,7 +191,7 @@ jobs:
               --compare-branch=origin/${{ env.COMPARE_BRANCH }}  | grep "No lines");
             echo DIFF_BODY="$DIFF_BODY_INTRO $DIFF_INFO">> $GITHUB_ENV
           else
-            if [[ $DIFF < 50 ]]; then COLOR=red; elif [[ $DIFF < 80 ]]; then COLOR=yellow; else COLOR=green; fi
+            if [[ $DIFF -lt 50 ]]; then COLOR=red; elif [[ $DIFF -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
             echo DIFF_BODY="$DIFF_BODY_INTRO ![Coverage-$DIFF%](https://img.shields.io/badge/coverage-$DIFF%25-$COLOR)" >> $GITHUB_ENV
           fi
 

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: check-ast
+    -   id: check-docstring-first
+        exclude: ^tests/
+    -   id: check-json
+    -   id: check-merge-conflict
+    -   id: check-yaml
+        args: ['--allow-multiple-documents']
+    -   id: detect-private-key
+    -   id: end-of-file-fixer
+    -   id: pretty-format-json
+        args: ['--autofix']
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.9.0
+    hooks:
+    -   id: rst-backticks
+    -   id: rst-inline-touching-normal
+
+exclude: ^(docs/)

ads/__init__.py

@@ -122,13 +122,8 @@ def _set_test_mode(mode=False):
 
 
 def hello():
-    """
-    Imports Pandas, sets the documentation mode, and prints a fancy "Hello".
-    """
-    import pandas
-
-    global documentation_mode
-    global debug_mode
+    import oci
+    import ocifs
 
     print(
         f"""
@@ -139,11 +134,11 @@ def hello():
 |   ||  /     |
 o   oo-o  o--o
 
-ADS SDK version: {__version__}
-Pandas version: {pandas.__version__}
-Debug mode: {debug_mode}
-"""
-    )
+ads v{__version__}
+oci v{oci.__version__}
+ocifs v{ocifs.__version__}
+
+""")
 
 
 configure_plotting()

ads/ads

@@ -1,2 +1,2 @@
 export OCI_PYTHON_SDK_NO_SERVICE_IMPORTS=1
-python -m ads.cli "$@"
+python3 -m ads.cli "$@"

ads/cli.py

@@ -3,17 +3,23 @@
 
 # Copyright (c) 2021, 2022 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from ads.common import logger
+import traceback
+
 try:
     import click
     import ads.opctl.cli
     import ads.jobs.cli
     import ads.pipeline.cli
     import os
     import json
-except:
+except Exception as ex:
     print(
-        "Please run `pip install oracle-ads[opctl]` to install the required dependencies for ADS CLI"
+        "Please run `pip install oracle-ads[opctl]` to install "
+        "the required dependencies for ADS CLI."
     )
+    logger.debug(ex)
+    logger.debug(traceback.format_exc())
     exit()
 
 

ads/common/extended_enum.py

@@ -6,6 +6,7 @@
 
 
 from abc import ABCMeta
+from enum import Enum
 
 
 class ExtendedEnumMeta(ABCMeta):
@@ -15,13 +16,20 @@ class ExtendedEnumMeta(ABCMeta):
     -------
     values(cls) -> list:
         Gets the list of class attributes.
+
+    Examples
+    --------
+    >>> class TestEnum(str, metaclass=ExtendedEnumMeta):
+    ...    KEY1 = "value1"
+    ...    KEY2 = "value2"
+    >>> print(TestEnum.KEY1) # "value1"
     """
 
     def __contains__(cls, value):
         return value and value.lower() in tuple(value.lower() for value in cls.values())
 
     def values(cls) -> list:
-        """Gets the list of class attributes.
+        """Gets the list of class attributes values.
 
         Returns
         -------
@@ -31,3 +39,35 @@ def values(cls) -> list:
         return tuple(
             value for key, value in cls.__dict__.items() if not key.startswith("_")
         )
+
+    def keys(cls) -> list:
+        """Gets the list of class attributes names.
+
+        Returns
+        -------
+        list
+            The list of class attributes names.
+        """
+        return tuple(
+            key for key, value in cls.__dict__.items() if not key.startswith("_")
+        )
+
+
+class ExtendedEnum(Enum):
+    """The base class to extend functionality of a generic Enum.
+
+    Examples
+    --------
+    >>> class TestEnum(ExtendedEnumMeta):
+    ...    KEY1 = "value1"
+    ...    KEY2 = "value2"
+    >>> print(TestEnum.KEY1.value) # "value1"
+    """
+
+    @classmethod
+    def values(cls):
+        return sorted(map(lambda c: c.value, cls))
+
+    @classmethod
+    def keys(cls):
+        return sorted(map(lambda c: c.name, cls))

ads/common/object_storage_details.py

@@ -121,3 +121,20 @@ def is_valid_uri(uri: str) -> bool:
                 "It must follow the pattern `oci://<bucket_name>@<namespace>/<prefix>`."
             )
         return True
+
+    @staticmethod
+    def is_oci_path(uri: str = None) -> bool:
+        """Check if the given path is oci object storage uri.
+
+        Parameters
+        ----------
+        uri: str
+            The URI of the target.
+
+        Returns
+        -------
+        bool: return True if the path is oci object storage uri.
+        """
+        if not uri:
+            return False
+        return uri.startswith("oci://")