Merge commit '09d471ae1329e5093234a725a0aa39321d8fbf43' from v2.9.0

Author: qiuosier

.github/workflows/run-unittests-py38-cov-report.yml

@@ -61,6 +61,12 @@ jobs:
         name: "Test env setup"
         timeout-minutes: 20
 
+      # Installing forecast deps for python3.8 test setup only, it will not work with python3.9/3.10, because
+      # automlx do not support py3.9 and some versions of py3.10. This step omitted in -py39-py30.yml workflow
+      - name: "Install Forecasting dependencies"
+        run: |
+          pip install -e ".[forecast]"
+
       - name: "Run unitary tests folder with maximum ADS dependencies"
         timeout-minutes: 60
         shell: bash

.github/workflows/run-unittests-py39-py310.yml

@@ -36,7 +36,7 @@ jobs:
         test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
         include:
           - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
-            ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
+            ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store --ignore tests/unitary/with_extras/operator/forecast"
             name: "unitary"
           - test-path: "tests/unitary/with_extras/model"
             name: "model"

.gitignore

@@ -160,3 +160,6 @@ logs/
 
 # vim
 *.swp
+
+# Python Wheel
+*.whl

MANIFEST.in

@@ -24,3 +24,7 @@ exclude build/lib/notebooks/**
 exclude benchmark/**
 include ads/ads
 include ads/model/common/*.*
+include ads/operator/**/*.md
+include ads/operator/**/*.yaml
+include ads/operator/**/*.whl
+include ads/operator/**/MLoperator

README.md

@@ -27,6 +27,14 @@ You have various options when installing ADS.
   python3 -m pip install oracle-ads
 ```
 
+### Installing OCI AI Operators
+
+To use the AI Forecast Operator, install the "forecast" dependencies using the following command:
+
+```bash
+  python3 -m pip install 'oracle_ads[forecast]==2.9.0'
+```
+
 ### Installing extras libraries
 
 To work with gradient boosting models, install the `boosted` module. This module includes XGBoost and LightGBM model classes.

ads/cli.py

@@ -14,12 +14,12 @@
     import ads.opctl.cli
     import ads.jobs.cli
     import ads.pipeline.cli
-    import os
-    import json
+    import ads.opctl.operator.cli
 except Exception as ex:
     print(
         "Please run `pip install oracle-ads[opctl]` to install "
-        "the required dependencies for ADS CLI."
+        "the required dependencies for ADS CLI. \n"
+        f"{str(ex)}"
     )
     logger.debug(ex)
     logger.debug(traceback.format_exc())
@@ -44,6 +44,7 @@ def cli():
 cli.add_command(ads.opctl.cli.commands)
 cli.add_command(ads.jobs.cli.commands)
 cli.add_command(ads.pipeline.cli.commands)
+cli.add_command(ads.opctl.operator.cli.commands)
 
 
 if __name__ == "__main__":

ads/common/auth.py

@@ -629,8 +629,8 @@ def create_signer(self) -> Dict:
                 user=configuration["user"],
                 fingerprint=configuration["fingerprint"],
                 private_key_file_location=configuration.get("key_file"),
-                pass_phrase= configuration.get("pass_phrase"),
-                private_key_content=configuration.get("key_content")
+                pass_phrase=configuration.get("pass_phrase"),
+                private_key_content=configuration.get("key_content"),
             ),
             "client_kwargs": self.client_kwargs,
         }
@@ -750,21 +750,10 @@ class SecurityToken(AuthSignerGenerator):
     a given user - it requires that user's private key and security token.
     It prepares extra arguments necessary for creating clients for variety of OCI services.
     """
-    SECURITY_TOKEN_GENERIC_HEADERS = [
-        "date",
-        "(request-target)",
-        "host"
-    ]
-    SECURITY_TOKEN_BODY_HEADERS = [
-        "content-length",
-        "content-type",
-        "x-content-sha256"
-    ]
-    SECURITY_TOKEN_REQUIRED = [
-        "security_token_file",
-        "key_file",
-        "region"
-    ]
+
+    SECURITY_TOKEN_GENERIC_HEADERS = ["date", "(request-target)", "host"]
+    SECURITY_TOKEN_BODY_HEADERS = ["content-length", "content-type", "x-content-sha256"]
+    SECURITY_TOKEN_REQUIRED = ["security_token_file", "key_file", "region"]
 
     def __init__(self, args: Optional[Dict] = None):
         """
@@ -831,12 +820,18 @@ def create_signer(self) -> Dict:
         return {
             "config": configuration,
             "signer": oci.auth.signers.SecurityTokenSigner(
-                token=self._read_security_token_file(configuration.get("security_token_file")),
+                token=self._read_security_token_file(
+                    configuration.get("security_token_file")
+                ),
                 private_key=oci.signer.load_private_key_from_file(
                     configuration.get("key_file"), configuration.get("pass_phrase")
                 ),
-                generic_headers=configuration.get("generic_headers", self.SECURITY_TOKEN_GENERIC_HEADERS),
-                body_headers=configuration.get("body_headers", self.SECURITY_TOKEN_BODY_HEADERS)
+                generic_headers=configuration.get(
+                    "generic_headers", self.SECURITY_TOKEN_GENERIC_HEADERS
+                ),
+                body_headers=configuration.get(
+                    "body_headers", self.SECURITY_TOKEN_BODY_HEADERS
+                ),
             ),
             "client_kwargs": self.client_kwargs,
         }
@@ -849,30 +844,37 @@ def _validate_and_refresh_token(self, configuration: Dict[str, Any]):
         configuration: Dict
             Security token configuration.
         """
-        security_token = self._read_security_token_file(configuration.get("security_token_file"))
-        security_token_container = oci.auth.security_token_container.SecurityTokenContainer(
-            session_key_supplier=None,
-            security_token=security_token
+        security_token = self._read_security_token_file(
+            configuration.get("security_token_file")
+        )
+        security_token_container = (
+            oci.auth.security_token_container.SecurityTokenContainer(
+                session_key_supplier=None, security_token=security_token
+            )
         )
 
         if not security_token_container.valid():
             raise SecurityTokenError(
                 "Security token has expired. Call `oci session authenticate` to generate new session."
             )
-        
+
         time_now = int(time.time())
         time_expired = security_token_container.get_jwt()["exp"]
         if time_expired - time_now < SECURITY_TOKEN_LEFT_TIME:
             if not self.oci_config_location:
-                logger.warning("Can not auto-refresh token. Specify parameter `oci_config_location` through ads.set_auth() or ads.auth.create_signer().")
+                logger.warning(
+                    "Can not auto-refresh token. Specify parameter `oci_config_location` through ads.set_auth() or ads.auth.create_signer()."
+                )
             else:
-                result = os.system(f"oci session refresh --config-file {self.oci_config_location} --profile {self.oci_key_profile}")
+                result = os.system(
+                    f"oci session refresh --config-file {self.oci_config_location} --profile {self.oci_key_profile}"
+                )
                 if result == 1:
                     logger.warning(
                         "Some error happened during auto-refreshing the token. Continue using the current one that's expiring in less than {SECURITY_TOKEN_LEFT_TIME} seconds."
                         "Please follow steps in https://docs.oracle.com/en-us/iaas/Content/API/SDKDocs/clitoken.htm to renew token."
                     )
-        
+
         date_time = datetime.fromtimestamp(time_expired).strftime("%Y-%m-%d %H:%M:%S")
         logger.info(f"Session is valid until {date_time}.")
 
@@ -894,7 +896,7 @@ def _read_security_token_file(self, security_token_file: str) -> str:
             raise ValueError("Invalid `security_token_file`. Specify a valid path.")
         try:
             token = None
-            with open(expanded_path, 'r') as f:
+            with open(expanded_path, "r") as f:
                 token = f.read()
             return token
         except:
@@ -903,7 +905,7 @@ def _read_security_token_file(self, security_token_file: str) -> str:
 
 class AuthFactory:
     """
-    AuthFactory class which contains list of registered signers and alllows to register new signers.
+    AuthFactory class which contains list of registered signers and allows to register new signers.
     Check documentation for more signers: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html.
 
     Current signers:

ads/common/decorator/runtime_dependency.py

@@ -64,6 +64,8 @@ class OptionalDependency:
     OPTUNA = "oracle-ads[optuna]"
     SPARK = "oracle-ads[spark]"
     HUGGINGFACE = "oracle-ads[huggingface]"
+    FORECAST = "oracle-ads[forecast]"
+    PII = "oracle-ads[pii]"
     FEATURE_STORE = "oracle-ads[feature-store]"
     GRAPHVIZ = "oracle-ads[graphviz]"
     MLM_INSIGHTS = "oracle-ads[mlm_insights]"

ads/common/object_storage_details.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json
@@ -15,7 +15,7 @@
 from ads.common import oci_client
 
 
-class InvalidObjectStoragePath(Exception):   # pragma: no cover
+class InvalidObjectStoragePath(Exception):  # pragma: no cover
     """Invalid Object Storage Path."""
 
     pass
@@ -137,4 +137,4 @@ def is_oci_path(uri: str = None) -> bool:
         """
         if not uri:
             return False
-        return uri.startswith("oci://")
+        return uri.lower().startswith("oci://")