Merge branch 'feature/forecast_v2'

Author: ahosler

.github/workflows/run-operators-unit-tests.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: python ${{ matrix.python-version }}
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 180
 
     strategy:
       fail-fast: false
@@ -49,7 +49,7 @@ jobs:
         name: "Test config setup"
 
       - name: "Run Operators Tests"
-        timeout-minutes: 60
+        timeout-minutes: 180
         shell: bash
         run: |
           set -x # print commands that are executed

ads/common/auth.py

@@ -687,6 +687,13 @@ def create_signer(self) -> Dict:
         logger.info(f"Using 'resource_principal' authentication.")
         return signer_dict
 
+    @staticmethod
+    def supported():
+        return any(
+            os.environ.get(var)
+            for var in ['JOB_RUN_OCID', 'NB_SESSION_OCID', 'DATAFLOW_RUN_ID', 'PIPELINE_RUN_OCID']
+        )
+
 
 class InstancePrincipal(AuthSignerGenerator):
     """

ads/opctl/config/merger.py

@@ -11,7 +11,7 @@
 
 import yaml
 
-from ads.common.auth import AuthType
+from ads.common.auth import AuthType, ResourcePrincipal
 from ads.opctl import logger
 from ads.opctl.config.base import ConfigProcessor
 from ads.opctl.config.utils import read_from_ini, _DefaultNoneDict
@@ -115,7 +115,7 @@ def _fill_config_with_defaults(self, ads_config_path: str) -> None:
         )
         # set default auth
         if not self.config["execution"].get("auth", None):
-            if is_in_notebook_session():
+            if ResourcePrincipal.supported():
                 self.config["execution"]["auth"] = (
                     exec_config.get("auth") or AuthType.RESOURCE_PRINCIPAL
                 )

ads/opctl/operator/__init__.py

@@ -14,7 +14,9 @@ def __registered_operators():
     return [
         f
         for f in os.listdir(target_dir)
-        if os.path.isdir(os.path.join(target_dir, f)) and not f.startswith("__")
+        if os.path.isdir(os.path.join(target_dir, f))
+        and not f.startswith("__")
+        and f != "common"
     ]
 
 

ads/opctl/operator/cli.py

@@ -9,12 +9,14 @@
 import click
 import fsspec
 import yaml
+import logging
 from ads.opctl.operator.common.utils import default_signer
 from ads.common.auth import AuthType
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl.constants import BACKEND_NAME, RUNTIME_TYPE
 from ads.opctl.decorator.common import click_options, with_auth, with_click_unknown_args
 from ads.opctl.utils import suppress_traceback
+from ads.opctl import logger
 
 from .__init__ import __operators__
 from .cmd import run as cmd_run
@@ -311,10 +313,14 @@ def publish_conda(debug: bool, **kwargs: Dict[str, Any]) -> None:
 @click.pass_context
 @with_click_unknown_args
 @with_auth
-def run(ctx: click.core.Context, debug: bool, **kwargs: Dict[str, Any]) -> None:
+def run(ctx: click.core.Context, debug: bool = False, **kwargs: Dict[str, Any]) -> None:
     """
     Runs the operator with the given specification on the targeted backend.
     """
+    if debug:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.CRITICAL)
     operator_spec = {}
     backend = kwargs.pop("backend")
 

ads/opctl/operator/cmd.py

@@ -48,7 +48,7 @@
 from .common.errors import (
     OperatorCondaNotFoundError,
     OperatorImageNotFoundError,
-    OperatorSchemaYamlError,
+    InvalidParameterError,
 )
 from .common.operator_loader import _operator_info_list
 
@@ -167,7 +167,7 @@ def init(
             )
     else:
         overwrite = True
-        output = os.path.join(tempfile.TemporaryDirectory().name, "")
+        output = operator_utils.create_output_folder(name=type + "/")
 
     # generating operator specification
     operator_config = {}
@@ -422,7 +422,7 @@ def verify(
             run_name="verify",
         )
         operator_module.get("verify")(config, **kwargs)
-    except OperatorSchemaYamlError as ex:
+    except InvalidParameterError as ex:
         logger.debug(ex)
         raise ValueError(
             f"The operator's specification is not valid for the `{operator_info.type}` operator. "

ads/opctl/operator/common/errors.py

@@ -7,8 +7,9 @@
 from ads.opctl.operator import __operators__
 
 
-class OperatorSchemaYamlError(Exception):
+class InvalidParameterError(Exception):
     """Exception raised when there is an issue with the schema."""
+
     def __init__(self, error: str):
         super().__init__(
             "Invalid operator specification. Check the YAML structure and ensure it "

ads/opctl/operator/common/operator_config.py

@@ -8,12 +8,31 @@
 import json
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 from ads.common.serializer import DataClassSerializable
 
 from ads.opctl.operator.common.utils import OperatorValidator
-from ads.opctl.operator.common.errors import OperatorSchemaYamlError
+from ads.opctl.operator.common.errors import InvalidParameterError
+
+@dataclass(repr=True)
+class InputData(DataClassSerializable):
+    """Class representing operator specification input data details."""
+
+    connect_args: Dict = None
+    format: str = None
+    columns: List[str] = None
+    url: str = None
+    filters: List[str] = None
+    options: Dict = None
+    limit: int = None
+    sql: str = None
+    table_name: str = None
+
+
+@dataclass(repr=True)
+class OutputDirectory(InputData):
+    """Class representing operator specification output directory details."""
 
 
 @dataclass(repr=True)
@@ -65,7 +84,7 @@ def _validate_dict(cls, obj_dict: Dict) -> bool:
         result = validator.validate(obj_dict)
 
         if not result:
-            raise OperatorSchemaYamlError(json.dumps(validator.errors, indent=2))
+            raise InvalidParameterError(json.dumps(validator.errors, indent=2))
         return True
 
     @classmethod

ads/opctl/operator/common/utils.py

@@ -29,6 +29,22 @@ class OperatorValidator(Validator):
     pass
 
 
+def create_output_folder(name):
+    output_folder = name
+    protocol = fsspec.utils.get_protocol(output_folder)
+    storage_options = {}
+    if protocol != "file":
+        storage_options = auth or default_signer()
+
+    fs = fsspec.filesystem(protocol, **storage_options)
+    name_suffix = 1
+    while fs.exists(output_folder):
+        name_suffix = name_suffix + 1
+        output_folder = f"{name}_{name_suffix}"
+    fs.mkdirs(output_folder)
+    return output_folder
+
+
 def _build_image(
     dockerfile: str,
     image_name: str,

ads/opctl/operator/lowcode/anomaly/MLoperator

@@ -0,0 +1,15 @@
+type: anomaly
+version: v1
+conda_type: service
+name: Anomaly Detection Operator
+gpu: no
+keywords:
+  - Anomaly Detection
+backends:
+  - job
+  - operator.local
+description: |
+  Anomaly Detection is the identification of rare items, events, or observations in data that
+  differ significantly from the expectation. This can be used for several scenarios like asset
+  monitoring, maintenance and prognostic surveillance in industries such as utility,
+  aviation and manufacturing.