Merge commit '2060741013d876b62c730183eb8892c1e56d3488' from main

Author: qiuosier

.github/workflows/run-operators-unit-tests.yml

@@ -0,0 +1,59 @@
+name: "[Py3.8][Py3.10] Operators Tests"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "ads/opctl/operator/**"
+      - "**requirements.txt"
+      - ".github/workflows/run-operators*.yml"
+      - "test-requirements-operators.txt"
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.10.8"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
+            "test-requirements-operators.txt"
+
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
+
+      - name: "Run Operators Tests"
+        timeout-minutes: 60
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          $CONDA/bin/conda init
+          source /home/runner/.bashrc
+          pip install -r test-requirements-operators.txt
+          python -m pytest -v -p no:warnings --durations=5 tests/operators

.github/workflows/run-unittests-py38-cov-report.yml

@@ -32,13 +32,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
+        name: ["unitary", "slow_tests"]
         include:
-          - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
-            ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
-            name: "unitary"
-          - test-path: "tests/unitary/with_extras/model"
-            name: "model"
+          - name: "unitary"
+            test-path: "tests/unitary"
+            # `model` tests running in "slow_tests",
+            # `feature_store` tests has its own test suite
+            ignore-path: |
+              --ignore tests/unitary/with_extras/model \
+              --ignore tests/unitary/with_extras/feature_store \
+              --ignore tests/unitary/with_extras/hpo
+          - name: "slow_tests"
+            test-path: "tests/unitary/with_extras/model"
 
     steps:
       - uses: actions/checkout@v3
@@ -68,7 +73,7 @@ jobs:
           pip install -e ".[forecast]"
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
-        timeout-minutes: 60
+        timeout-minutes: 40
         shell: bash
         env:
           CONDA_PREFIX: /usr/share/miniconda
@@ -82,9 +87,17 @@ jobs:
           ln -s ../ads ads
           ln -s ../.coveragerc .coveragerc
 
+          # Run hpo tests, which hangs if run together with all unitary tests
+          if [[ ${{ matrix.name }} ==  "unitary" ]]; then
+            python -m pytest -v -p no:warnings -n auto --dist loadfile \
+            --cov-append --cov=ads --cov-report=xml --cov-report=html \
+            tests/unitary/with_extras/hpo
+          fi
+
           # Run tests
           python -m pytest -v -p no:warnings --durations=5 \
-          -n auto --dist loadfile --cov=ads --cov-report=xml --cov-report=html \
+          -n auto --dist loadfile \
+          --cov-append --cov=ads --cov-report=xml --cov-report=html \
           ${{ matrix.test-path }} ${{ matrix.ignore-path }}
 
       - name: "Save coverage files"
@@ -123,11 +136,11 @@ jobs:
           # Prepare file paths to .coverage files
           # Filenames taken from job.test last step with name - "Save coverage files"
           FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
-          FILE_MODEL="cov-reports-model/.coverage"; [[ ! -f $FILE_MODEL ]] && FILE_MODEL=""
+          FILE_SLOW="cov-reports-slow_tests/.coverage"; [[ ! -f $FILE_SLOW ]] && FILE_SLOW=""
 
           # Combine coverage files
           pip install coverage
-          coverage combine $FILE_UNITARY $FILE_MODEL
+          coverage combine $FILE_UNITARY $FILE_SLOW
 
           # Make html report
           coverage html
@@ -151,7 +164,7 @@ jobs:
           # Prepare file paths to coverage xml files
           # Filenames taken from job.test last step with name - "Save coverage files"
           FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
-          FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
+          FILE2="cov-reports-slow_tests/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
           echo "FILE1=$FILE1" >> $GITHUB_ENV
           echo "FILE2=$FILE2" >> $GITHUB_ENV
 

.github/workflows/run-unittests-py39-py310.yml

@@ -33,13 +33,21 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10"]
-        test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
+        name: ["unitary", "slow_tests"]
         include:
-          - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
-            ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store --ignore tests/unitary/with_extras/operator/forecast"
-            name: "unitary"
-          - test-path: "tests/unitary/with_extras/model"
-            name: "model"
+          - name: "unitary"
+            test-path: "tests/unitary"
+            # `model` tests running in "slow_tests",
+            # `feature_store` tests has its own test suite
+            # `forecast` tests not supported in python 3.9,3.10 (automlx dependency). Tests are running in python3.8 test env, see run-unittests-py38-cov-report.yml
+            # 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
+            ignore-path: |
+              --ignore tests/unitary/with_extras/model \
+              --ignore tests/unitary/with_extras/feature_store \
+              --ignore tests/unitary/with_extras/operator/forecast \
+              --ignore tests/unitary/with_extras/hpo
+          - name: "slow_tests"
+            test-path: "tests/unitary/with_extras/model"
 
     steps:
       - uses: actions/checkout@v3
@@ -62,8 +70,19 @@ jobs:
         name: "Test env setup"
         timeout-minutes: 20
 
+      - name: "Run hpo tests"
+        timeout-minutes: 10
+        shell: bash
+        if: ${{ matrix.name }} == "unitary"
+        run: |
+          set -x # print commands that are executed
+
+          # Run hpo tests, which hangs if run together with all unitary tests
+          python -m pytest -v -p no:warnings -n auto --dist loadfile \
+          tests/unitary/with_extras/hpo
+
       - name: "Run unitary tests folder with maximum ADS dependencies"
-        timeout-minutes: 60
+        timeout-minutes: 30
         shell: bash
         env:
           CONDA_PREFIX: /usr/share/miniconda

ads/common/auth.py

@@ -158,29 +158,29 @@ def set_auth(
     ... )
     >>> ads.set_auth("security_token", config=config) # Set security token authentication from provided config
 
-    >>> singer = oci.signer.Signer(
+    >>> signer = oci.signer.Signer(
     ...     user=ocid1.user.oc1..<unique_ID>,
     ...     fingerprint=<fingerprint>,
     ...     tenancy=ocid1.tenancy.oc1..<unique_ID>,
     ...     region=us-ashburn-1,
     ...     private_key_content=<private key content>,
     ... )
-    >>> ads.set_auth(singer=singer) # Set api keys authentication with private key content based on provided signer
+    >>> ads.set_auth(signer=signer) # Set api keys authentication with private key content based on provided signer
 
-    >>> singer = oci.signer.Signer(
+    >>> signer = oci.signer.Signer(
     ...     user=ocid1.user.oc1..<unique_ID>,
     ...     fingerprint=<fingerprint>,
     ...     tenancy=ocid1.tenancy.oc1..<unique_ID>,
     ...     region=us-ashburn-1,
     ...     private_key_file_location=<private key content>,
     ... )
-    >>> ads.set_auth(singer=singer) # Set api keys authentication with private key file location based on provided signer
+    >>> ads.set_auth(signer=signer) # Set api keys authentication with private key file location based on provided signer
 
-    >>> singer = oci.auth.signers.get_resource_principals_signer()
-    >>> ads.auth.create_signer(config={}, singer=signer) # resource principals authentication dictionary created
+    >>> signer = oci.auth.signers.get_resource_principals_signer()
+    >>> ads.auth.create_signer(config={}, signer=signer) # resource principals authentication dictionary created
 
     >>> signer_callable = oci.auth.signers.ResourcePrincipalsFederationSigner
-    >>> ads.set_auth(signer_callable=signer_callable) # Set resource principal federation singer callable
+    >>> ads.set_auth(signer_callable=signer_callable) # Set resource principal federation signer callable
 
     >>> signer_callable = oci.auth.signers.InstancePrincipalsSecurityTokenSigner
     >>> signer_kwargs = dict(log_requests=True) # will log the request url and response data when retrieving
@@ -400,7 +400,7 @@ def create_signer(
     ... }
     >>> auth = ads.auth.create_signer(config=config) # api_key type of authentication dictionary with private key file location created based on provided config
 
-    >>> singer = oci.auth.signers.get_resource_principals_signer()
+    >>> signer = oci.auth.signers.get_resource_principals_signer()
     >>> auth = ads.auth.create_signer(config={}, signer=signer) # resource principals authentication dictionary created
 
     >>> auth = ads.auth.create_signer(auth_type='instance_principal') # instance principals authentication dictionary created
@@ -517,7 +517,7 @@ def get_signer(
 ) -> Dict:
     """
     Provides config and signer based given parameters. If oci_config (api key config file location) and
-    oci_profile specified new signer will ge generated. Else singer of a type specified in OCI_CLI_AUTH
+    oci_profile specified new signer will ge generated. Else signer of a type specified in OCI_CLI_AUTH
     environment variable will be used to generate signer and return. If OCI_CLI_AUTH not set,
     resource principal signer will be provided. Accepted values for OCI_CLI_AUTH: 'api_key',
     'instance_principal', 'resource_principal'.
@@ -929,9 +929,9 @@ def register(cls, signer_type: str, signer: Any) -> None:
         Parameters
         ----------
         signer_type: str
-            Singer type to be registers
+            signer type to be registers
         signer: RecordParser
-            A new Singer class to be registered.
+            A new signer class to be registered.
 
         Returns
         -------

ads/common/oci_client.py

@@ -17,6 +17,7 @@
 from oci.resource_search import ResourceSearchClient
 from oci.secrets import SecretsClient
 from oci.vault import VaultsClient
+
 logger = logging.getLogger(__name__)
 
 
@@ -64,13 +65,14 @@ def _client_impl(self, client):
             "data_labeling_dp": DataLabelingClient,
             "data_labeling_cp": DataLabelingManagementClient,
             "resource_search": ResourceSearchClient,
-            "data_catalog": DataCatalogClient
+            "data_catalog": DataCatalogClient,
         }
         try:
             from oci.feature_store import FeatureStoreClient
+
             client_map["feature_store"] = FeatureStoreClient
         except ImportError:
-            logger.warning("OCI SDK with feature store support is not installed")
+            logger.debug("OCI SDK with feature store support is not installed")
             pass
 
         assert (

ads/opctl/config/merger.py

@@ -124,7 +124,10 @@ def _fill_config_with_defaults(self, ads_config_path: str) -> None:
                     exec_config.get("auth") or AuthType.API_KEY
                 )
         # determine profile
-        if self.config["execution"]["auth"] != AuthType.API_KEY:
+        if self.config["execution"]["auth"] in (
+            AuthType.RESOURCE_PRINCIPAL,
+            AuthType.INSTANCE_PRINCIPAL,
+        ):
             profile = self.config["execution"]["auth"].upper()
             exec_config.pop("oci_profile", None)
             self.config["execution"]["oci_profile"] = None

ads/opctl/constants.py

@@ -32,6 +32,7 @@
 # OPERATOR
 OPERATOR_MODULE_PATH = "ads.opctl.operator.lowcode"
 OPERATOR_IMAGE_WORK_DIR = "/etc/operator"
+OVERRIDE_KWARGS = "override_kwargs"
 
 
 class RUNTIME_TYPE(ExtendedEnum):

ads/opctl/decorator/common.py

@@ -7,10 +7,13 @@
 from functools import wraps
 from typing import Callable, Dict, List
 
+import click
+
 from ads.common.auth import AuthContext
 from ads.opctl import logger
 from ads.opctl.config.base import ConfigProcessor
 from ads.opctl.config.merger import ConfigMerger
+from ads.opctl.constants import OVERRIDE_KWARGS
 
 RUN_ID_FIELD = "run_id"
 
@@ -101,3 +104,26 @@ def wrapper(*args, **kwargs) -> Dict:
             return func(*args, **kwargs)
 
     return wrapper
+
+
+def with_click_unknown_args(func: Callable) -> Callable:
+    """The decorator to parse the click unknown arguments and put them into kwargs."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs) -> Dict:
+        kwargs[OVERRIDE_KWARGS] = {}
+        try:
+            click_context = next(
+                item for item in args if isinstance(item, click.core.Context)
+            )
+            kwargs[OVERRIDE_KWARGS] = {
+                key[2:]: value
+                for key, value in zip(click_context.args[::2], click_context.args[1::2])
+            }
+        except Exception as ex:
+            logger.debug(ex)
+            pass
+
+        return func(*args, **kwargs)
+
+    return wrapper

ads/opctl/distributed/cmds.py

@@ -4,18 +4,22 @@
 # Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import os
-import fsspec
-import subprocess
-import yaml
 import json
+import os
 import re
-import docker
+import subprocess
 from configparser import ConfigParser
-
-from ads.common.auth import create_signer, AuthType, AuthContext
 from urllib.parse import urlparse
-from ads.jobs import Job, DataScienceJobRun
+
+import fsspec
+import yaml
+
+from ads.common.auth import AuthContext, AuthType, create_signer
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.jobs import Job
 from ads.jobs.builders.runtimes.artifact import Artifact
 from ads.opctl.utils import publish_image as publish_image_cmd
 from ads.opctl.utils import run_command
@@ -137,6 +141,7 @@ def show_config_info(job_id, work_dir, cluster_file_name, worker_info, **kwargs)
         )
 
 
+@runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
 def verify_image(img_name):
     """
     Verify if the input image exists in OCI registry
@@ -151,6 +156,8 @@ def verify_image(img_name):
     -------
 
     """
+    import docker
+
     client = docker.from_env()
     try:
         client.images.get_registry_data(img_name)