Merge branch 'main' into ODSC-59115/mlforecast_add_data

Author: ahosler

ads/opctl/operator/lowcode/anomaly/README.md

@@ -58,7 +58,7 @@ The operator will run in your local environment without requiring any additional
 
 ## 4. Running anomaly detection on the local container
 
-To run the anomaly detection detection operator within a local container, follow these steps:
+To run the anomaly detection operator within a local container, follow these steps:
 
 Use the command below to build the anomaly detection container.
 
@@ -106,7 +106,7 @@ ads operator run -f ~/anomaly/anomaly.yaml --backend-config ~/anomaly/backend_op
 
 ## 5. Running anomaly detection in the Data Science job within container runtime
 
-To execute the anomaly detection detection operator within a Data Science job using container runtime, please follow the steps outlined below:
+To execute the anomaly detection operator within a Data Science job using container runtime, please follow the steps outlined below:
 
 You can use the following command to build the anomaly detection container. This step can be skipped if you have already done this for running the operator within a local container.
 
@@ -155,7 +155,7 @@ ads opctl watch <OCID>
 
 ## 6. Running anomaly detection in the Data Science job within conda runtime
 
-To execute the anomaly detection detection operator within a Data Science job using conda runtime, please follow the steps outlined below:
+To execute the anomaly detection operator within a Data Science job using conda runtime, please follow the steps outlined below:
 
 You can use the following command to build the anomaly detection conda environment.
 

ads/opctl/operator/lowcode/anomaly/__main__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
@@ -15,7 +14,7 @@
 from ads.opctl.operator.common.const import ENV_OPERATOR_ARGS
 from ads.opctl.operator.common.utils import _parse_input_args
 
-from .model.anomaly_dataset import AnomalyDatasets, AnomalyData
+from .model.anomaly_dataset import AnomalyDatasets
 from .operator_config import AnomalyOperatorConfig
 
 
@@ -34,7 +33,7 @@ def operate(operator_config: AnomalyOperatorConfig) -> None:
                 f"Failed to forecast with error {e.args}. Trying again with model `autots`."
             )
             operator_config.spec.model = "autots"
-            operator_config.spec.model_kwargs = dict()
+            operator_config.spec.model_kwargs = {}
             datasets = AnomalyDatasets(operator_config.spec)
             try:
                 AnomalyOperatorModelFactory.get_model(
@@ -44,12 +43,12 @@ def operate(operator_config: AnomalyOperatorConfig) -> None:
                 logger.debug(
                     f"Failed to backup forecast with error {ee.args}. Raising original error."
                 )
-            raise ee
+                raise ee
         else:
             raise e
 
 
-def verify(spec: Dict, **kwargs: Dict) -> bool:
+def verify(spec: Dict) -> bool:
     """Verifies the anomaly detection operator config."""
     operator = AnomalyOperatorConfig.from_dict(spec)
     msg_header = (
@@ -83,7 +82,7 @@ def main(raw_args: List[str]):
             yaml_string = yaml.safe_dump(json.loads(operator_spec_str))
         except json.JSONDecodeError:
             yaml_string = yaml.safe_dump(yaml.safe_load(operator_spec_str))
-        except:
+        except Exception:
             yaml_string = operator_spec_str
 
     operator_config = AnomalyOperatorConfig.from_yaml(

ads/opctl/operator/lowcode/anomaly/const.py

@@ -16,6 +16,14 @@ class SupportedModels(str, metaclass=ExtendedEnumMeta):
     Auto = "auto"
     # TODS = "tods"
 
+class NonTimeADSupportedModels(str, metaclass=ExtendedEnumMeta):
+    """Supported non time-based anomaly detection models."""
+
+    OneClassSVM = "oneclasssvm"
+    IsolationForest = "isolationforest"
+    # TODO : Add DBScan
+    # DBScan = "dbscan"
+    
 
 class TODSSubModels(str, metaclass=ExtendedEnumMeta):
     """Supported TODS sub models."""

ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py

@@ -84,17 +84,21 @@ def get_inliers_by_cat(self, category: str, data: pd.DataFrame):
         scores = self.get_scores_by_cat(category)
         inlier_indices = anomaly.index[anomaly[OutputColumns.ANOMALY_COL] == 0]
         inliers = data.iloc[inlier_indices]
-        if scores is not None and not scores.empty:
+        if scores is not None and not scores.empty and self.date_column != "index":
             inliers = pd.merge(inliers, scores, on=self.date_column, how="inner")
+        else:
+            inliers = pd.merge(inliers, anomaly, left_index=True, right_index=True, how="inner")
         return inliers
 
     def get_outliers_by_cat(self, category: str, data: pd.DataFrame):
         anomaly = self.get_anomalies_by_cat(category)
         scores = self.get_scores_by_cat(category)
         outliers_indices = anomaly.index[anomaly[OutputColumns.ANOMALY_COL] == 1]
         outliers = data.iloc[outliers_indices]
-        if scores is not None and not scores.empty:
+        if scores is not None and not scores.empty and self.date_column != "index":
             outliers = pd.merge(outliers, scores, on=self.date_column, how="inner")
+        else:
+            outliers = pd.merge(outliers, anomaly, left_index=True, right_index=True, how="inner")
         return outliers
 
     def get_inliers(self, datasets):

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -1,32 +1,33 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import fsspec
-import numpy as np
 import os
-import pandas as pd
 import tempfile
 import time
 from abc import ABC, abstractmethod
-from sklearn import linear_model
 from typing import Tuple
 
+import fsspec
+import numpy as np
+import pandas as pd
+from sklearn import linear_model
+
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl import logger
 from ads.opctl.operator.lowcode.anomaly.const import OutputColumns, SupportedMetrics
 from ads.opctl.operator.lowcode.anomaly.utils import _build_metrics_df, default_signer
 from ads.opctl.operator.lowcode.common.utils import (
-    human_time_friendly,
-    enable_print,
     disable_print,
+    enable_print,
+    human_time_friendly,
     write_data,
 )
-from .anomaly_dataset import AnomalyDatasets, AnomalyOutput, TestData
-from ..const import SupportedModels
+
+from ..const import NonTimeADSupportedModels, SupportedModels
 from ..operator_config import AnomalyOperatorConfig, AnomalyOperatorSpec
+from .anomaly_dataset import AnomalyDatasets, AnomalyOutput, TestData
 
 
 class AnomalyOperatorBaseModel(ABC):
@@ -53,15 +54,18 @@ def __init__(self, config: AnomalyOperatorConfig, datasets: AnomalyDatasets):
 
     def generate_report(self):
         """Generates the report."""
-        import report_creator as rc
         import matplotlib.pyplot as plt
+        import report_creator as rc
 
         start_time = time.time()
         # fallback using sklearn oneclasssvm when the sub model _build_model fails
         try:
             anomaly_output = self._build_model()
         except Exception as e:
-            anomaly_output = self._fallback_build_model()
+            logger.warn(f"Found exception: {e}")
+            if self.spec.datetime_column:
+                anomaly_output = self._fallback_build_model()
+            raise e
 
         elapsed_time = time.time() - start_time
 
@@ -79,7 +83,9 @@ def generate_report(self):
             for col, df in self.datasets.full_data_dict.items()
         ]
         data_table = rc.Select(blocks=table_blocks)
-        date_column = self.spec.datetime_column.name
+        date_column = (
+            self.spec.datetime_column.name if self.spec.datetime_column else "index"
+        )
 
         blocks = []
         for target, df in self.datasets.full_data_dict.items():
@@ -95,7 +101,7 @@ def generate_report(self):
                 ax.grid()
                 ax.plot(time_col, y, color="black")
                 for i, index in enumerate(anomaly_col):
-                    if anomaly_col[i] == 1:
+                    if index == 1:
                         ax.scatter(time_col[i], y[i], color="red", marker="o")
                 plt.xlabel(date_column)
                 plt.ylabel(col)
@@ -114,7 +120,7 @@ def generate_report(self):
                 rc.Text(f"You selected the **`{self.spec.model}`** model."),
                 rc.Text(
                     "Based on your dataset, you could have also selected "
-                    f"any of the models: `{'`, `'.join(SupportedModels.keys())}`."
+                    f"any of the models: `{'`, `'.join(SupportedModels.keys() if self.spec.datetime_column else NonTimeADSupportedModels.keys())}`."
                 ),
                 rc.Metric(
                     heading="Analysis was completed in ",
@@ -170,7 +176,9 @@ def _test_data_evaluate_metrics(self, anomaly_output, test_data, elapsed_time):
 
         for cat in anomaly_output.list_categories():
             output = anomaly_output.category_map[cat][0]
-            date_col = self.spec.datetime_column.name
+            date_col = (
+                self.spec.datetime_column.name if self.spec.datetime_column else "index"
+            )
 
             test_data_i = test_data.get_data_for_series(cat)
 
@@ -247,7 +255,7 @@ def _save_report(
         if ObjectStorageDetails.is_oci_path(unique_output_dir):
             storage_options = default_signer()
         else:
-            storage_options = dict()
+            storage_options = {}
 
         # report-creator html report
         with tempfile.TemporaryDirectory() as temp_dir:
@@ -301,12 +309,11 @@ def _fallback_build_model(self):
         Fallback method for the sub model _build_model method.
         """
         logger.warn(
-            "The build_model method has failed for the model: {}. "
-            "A fallback model will be built.".format(self.spec.model)
+            f"The build_model method has failed for the model: {self.spec.model}. "
+            "A fallback model will be built."
         )
 
         date_column = self.spec.datetime_column.name
-        dataset = self.datasets
 
         anomaly_output = AnomalyOutput(date_column=date_column)
 
@@ -320,7 +327,9 @@ def _fallback_build_model(self):
             y_pred = np.vectorize(self.outlier_map.get)(
                 est.predict(df[self.spec.target_column].fillna(0).values.reshape(-1, 1))
             )
-            scores = est.score_samples(df[self.spec.target_column].fillna(0).values.reshape(-1, 1))
+            scores = est.score_samples(
+                df[self.spec.target_column].fillna(0).values.reshape(-1, 1)
+            )
 
             anomaly = pd.DataFrame(
                 {date_column: df[date_column], OutputColumns.ANOMALY_COL: y_pred}

ads/opctl/operator/lowcode/anomaly/model/factory.py

@@ -1,26 +1,41 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-from ..const import SupportedModels
+from ads.opctl.operator.lowcode.anomaly.utils import select_auto_model
+
+from ..const import NonTimeADSupportedModels, SupportedModels
 from ..operator_config import AnomalyOperatorConfig
+from .anomaly_dataset import AnomalyDatasets
 from .automlx import AutoMLXOperatorModel
 from .autots import AutoTSOperatorModel
-from ads.opctl.operator.lowcode.anomaly.utils import select_auto_model
 
 # from .tods import TODSOperatorModel
 from .base_model import AnomalyOperatorBaseModel
-from .anomaly_dataset import AnomalyDatasets
+from .isolationforest import IsolationForestOperatorModel
+from .oneclasssvm import OneClassSVMOperatorModel
 
 
 class UnSupportedModelError(Exception):
-    def __init__(self, model_type: str):
-        super().__init__(
-            f"Model: `{model_type}` "
-            f"is not supported. Supported models: {SupportedModels.values}"
+    """Exception raised when the model is not supported.
+
+    Attributes:
+        operator_config (AnomalyOperatorConfig): The operator configuration.
+        model_type (str): The type of the unsupported model.
+    """
+
+    def __init__(self, operator_config: AnomalyOperatorConfig, model_type: str):
+        supported_models = (
+            SupportedModels.values
+            if operator_config.spec.datetime_column
+            else NonTimeADSupportedModels.values
         )
+        message = (
+            f"Model: `{model_type}` is not supported. "
+            f"Supported models: {supported_models}"
+        )
+        super().__init__(message)
 
 
 class AnomalyOperatorModelFactory:
@@ -34,6 +49,13 @@ class AnomalyOperatorModelFactory:
         SupportedModels.AutoTS: AutoTSOperatorModel,
     }
 
+    _NonTime_MAP = {
+        NonTimeADSupportedModels.OneClassSVM: OneClassSVMOperatorModel,
+        NonTimeADSupportedModels.IsolationForest: IsolationForestOperatorModel,
+        # TODO: Add DBScan model for non time based anomaly
+        # NonTimeADSupportedModels.DBScan: DBScanOperatorModel,
+    }
+
     @classmethod
     def get_model(
         cls, operator_config: AnomalyOperatorConfig, datasets: AnomalyDatasets
@@ -61,7 +83,13 @@ def get_model(
         """
         model_type = operator_config.spec.model
         if model_type == "auto":
-            model_type = select_auto_model(datasets, operator_config)
-        if model_type not in cls._MAP:
-            raise UnSupportedModelError(model_type)
-        return cls._MAP[model_type](config=operator_config, datasets=datasets)
+            model_type = select_auto_model(operator_config)
+
+        model_map = (
+            cls._MAP if operator_config.spec.datetime_column else cls._NonTime_MAP
+        )
+
+        if model_type not in model_map:
+            raise UnSupportedModelError(operator_config, model_type)
+
+        return model_map[model_type](config=operator_config, datasets=datasets)