Forecast V2 Fast Explainability (#501)

Author: prasankh

ads/opctl/operator/lowcode/forecast/const.py

@@ -6,7 +6,6 @@
 
 from ads.common.extended_enum import ExtendedEnumMeta
 
-
 class SupportedModels(str, metaclass=ExtendedEnumMeta):
     """Supported forecast models."""
 
@@ -18,6 +17,18 @@ class SupportedModels(str, metaclass=ExtendedEnumMeta):
     Auto = "auto"
 
 
+class SpeedAccuracyMode(str, metaclass=ExtendedEnumMeta):
+    """
+    Enum representing different modes based on time taken and accuracy for explainability.
+    """
+    HIGH_ACCURACY = "HIGH_ACCURACY"
+    BALANCED = "BALANCED"
+    FAST_APPROXIMATE = "FAST_APPROXIMATE"
+    ratio = dict()
+    ratio[HIGH_ACCURACY] = 1  # 100 % data used for generating explanations
+    ratio[BALANCED] = 0.5  # 50 % data used for generating explanations
+    ratio[FAST_APPROXIMATE] = 0   # constant
+
 class SupportedMetrics(str, metaclass=ExtendedEnumMeta):
     """Supported forecast metrics."""
 

ads/opctl/operator/lowcode/forecast/model/arima.py

@@ -251,6 +251,9 @@ def _custom_predict_arima(self, data):
             array-like: The predicted values.
 
         """
+        date_col = self.spec.datetime_column.name
+        data[date_col] = pd.to_datetime(data[date_col], unit='s')
+        data = data.set_index(date_col)
         # Get the index of the current series id
         series_index = self.target_columns.index(self.series_id)
 

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -316,7 +316,6 @@ def _custom_predict_automlx(self, data):
         -------
             numpy.ndarray: The predicted future values of the time series.
         """
-        temp = 0
         data_temp = pd.DataFrame(
             data,
             columns=[col for col in self.dataset_cols],

ads/opctl/operator/lowcode/forecast/model/autots.py

@@ -328,41 +328,7 @@ def _generate_report(self) -> tuple:
         )
 
     def _custom_predict_autots(self, data):
-        """
-        Predicts the future values of a time series using the AutoTS model.
-
-        Parameters
-        ----------
-            data (numpy.ndarray): The input data to be used for prediction.
-
-        Returns
-        -------
-            numpy.ndarray: The predicted future values of the time series.
-        """
-
-        data.index = pd.to_datetime(data.index)
-        temp_model = copy.deepcopy(self.models)
-
-        if data.shape[0] > 1:
-            temp_model.fit_data(
-                data[~data.index.duplicated()],
-                future_regressor=self.future_regressor_train.head(-self.spec.horizon),
-            )
-            dedup_shape = data.shape[0] - data[~data.index.duplicated()].shape[0] + 1
-            return pd.Series(0, index=np.arange(dedup_shape)).append(
-                temp_model.back_forecast(
-                    tail=data[~data.index.duplicated()].shape[0] - 1
-                )
-                .forecast[self.spec.target_column]
-                .fillna(0)
-            )
-
-        return temp_model.predict(
-            future_regressor=self.future_regressor_train.loc[
-                self.future_regressor_train.index.isin(data.index)
-            ],
-            forecast_length=1,
-        ).forecast[self.series_id]
+        raise NotImplementedError("Autots does not yet support explanations.")
 
     def _generate_train_metrics(self) -> pd.DataFrame:
         """

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -20,7 +20,7 @@
 from ads.opctl import logger
 
 from .. import utils
-from ..const import SUMMARY_METRICS_HORIZON_LIMIT, SupportedMetrics, SupportedModels
+from ..const import SUMMARY_METRICS_HORIZON_LIMIT, SupportedMetrics, SupportedModels, SpeedAccuracyMode
 from ..operator_config import ForecastOperatorConfig, ForecastOperatorSpec
 from ads.common.decorator.runtime_dependency import runtime_dependency
 from .forecast_datasets import ForecastDatasets, ForecastOutput
@@ -73,10 +73,10 @@ def generate_report(self):
             warnings.simplefilter(action="ignore", category=ConvergenceWarning)
             import datapane as dp
 
-            # load data and build models
             start_time = time.time()
             result_df = self._build_model()
             elapsed_time = time.time() - start_time
+            logger.info("Building the models completed in %s seconds", elapsed_time)
 
             # Generate metrics
             summary_metrics = None
@@ -574,49 +574,37 @@ def explain_model(self, datetime_col_name, explain_predict_fn) -> dict:
             dict: A dictionary containing the global explanation for each feature in the dataset.
                     The keys are the feature names and the values are the average absolute SHAP values.
         """
-        from shap import KernelExplainer
-
+        from shap import PermutationExplainer
+        exp_start_time = time.time()
+        global_ex_time = 0
+        local_ex_time = 0
+        logger.info(f"Calculating explanations using {self.spec.explanations_accuracy_mode} mode")
         for series_id in self.target_columns:
             self.series_id = series_id
-            if self.spec.model == SupportedModels.AutoTS:
-                self.dataset_cols = (
-                    self.full_data_long.loc[
-                        self.full_data_long.series_id == self.series_id
-                    ]
-                    .set_index(datetime_col_name)
-                    .columns
-                )
-
-                self.bg_data = self.full_data_long.loc[
-                    self.full_data_long.series_id == self.series_id
-                ].set_index(datetime_col_name)
-
-            else:
-                self.dataset_cols = (
-                    self.full_data_dict.get(series_id)
-                    .set_index(datetime_col_name)
-                    .drop(series_id, axis=1)
-                    .columns
-                )
-
-                self.bg_data = self.full_data_dict.get(series_id).set_index(
-                    datetime_col_name
-                )
+            self.dataset_cols = (
+                self.full_data_dict.get(series_id)
+                .set_index(datetime_col_name)
+                .drop(series_id, axis=1)
+                .columns
+            )
 
-            kernel_explnr = KernelExplainer(
+            self.bg_data = self.full_data_dict.get(series_id).set_index(
+                datetime_col_name
+            )
+            data = self.bg_data[list(self.dataset_cols)][: -self.spec.horizon][
+                list(self.dataset_cols)]
+            ratio = SpeedAccuracyMode.ratio[self.spec.explanations_accuracy_mode]
+            data_trimmed = data.tail(max(int(len(data) * ratio), 100)).reset_index()
+            data_trimmed[datetime_col_name] = data_trimmed[datetime_col_name].apply(lambda x: x.timestamp())
+            kernel_explnr = PermutationExplainer(
                 model=explain_predict_fn,
-                data=self.bg_data[list(self.dataset_cols)][: -self.spec.horizon][
-                    list(self.dataset_cols)
-                ],
+                masker=data_trimmed,
                 keep_index=False
                 if self.spec.model == SupportedModels.AutoMLX
                 else True,
             )
 
-            kernel_explnr_vals = kernel_explnr.shap_values(
-                self.bg_data[: -self.spec.horizon][list(self.dataset_cols)],
-                nsamples=50,
-            )
+            kernel_explnr_vals = kernel_explnr.shap_values(data_trimmed)
 
             if not len(kernel_explnr_vals):
                 logger.warn(
@@ -625,14 +613,19 @@ def explain_model(self, datetime_col_name, explain_predict_fn) -> dict:
             else:
                 self.global_explanation[series_id] = dict(
                     zip(
-                        self.dataset_cols,
-                        np.average(np.absolute(kernel_explnr_vals), axis=0),
+                        data_trimmed.columns[1:],
+                        np.average(np.absolute(kernel_explnr_vals[:, 1:]), axis=0),
                     )
                 )
+            exp_end_time = time.time()
+            global_ex_time = global_ex_time + exp_end_time - exp_start_time
 
             self.local_explainer(
                 kernel_explnr, series_id=series_id, datetime_col_name=datetime_col_name
             )
+            local_ex_time = local_ex_time + time.time() - exp_end_time
+        logger.info("Global explanations generation completed in %s seconds", global_ex_time)
+        logger.info("Local explanations generation completed in %s seconds", local_ex_time)
 
     def local_explainer(self, kernel_explainer, series_id, datetime_col_name) -> None:
         """
@@ -644,18 +637,19 @@ def local_explainer(self, kernel_explainer, series_id, datetime_col_name) -> Non
         """
         # Get the data for the series ID and select the relevant columns
         # data = self.full_data_dict.get(series_id).set_index(datetime_col_name)
-        data = self.bg_data[-self.spec.horizon :][list(self.dataset_cols)]
-
+        data_horizon = self.bg_data[-self.spec.horizon:][list(self.dataset_cols)]
+        data = data_horizon.reset_index()
+        data[datetime_col_name] = data[datetime_col_name].apply(lambda x: x.timestamp())
         # Generate local SHAP values using the kernel explainer
-        local_kernel_explnr_vals = kernel_explainer.shap_values(data, nsamples=50)
+        local_kernel_explnr_vals = kernel_explainer.shap_values(data)
 
         # Convert the SHAP values into a DataFrame
         local_kernel_explnr_df = pd.DataFrame(
-            local_kernel_explnr_vals, columns=self.dataset_cols
+            local_kernel_explnr_vals[:, 1:], columns=data.columns[1:]
         )
 
         # set the index of the DataFrame to the datetime column
-        local_kernel_explnr_df.index = data.index
+        local_kernel_explnr_df.index = data_horizon.index
 
         if self.spec.model == SupportedModels.AutoTS:
             local_kernel_explnr_df.drop(

ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py

@@ -12,7 +12,7 @@
 import pandas as pd
 from ..const import ForecastOutputColumns, PROPHET_INTERNAL_DATE_COL
 from pandas.api.types import is_datetime64_any_dtype, is_string_dtype, is_numeric_dtype
-
+import time
 
 class ForecastDatasets:
     def __init__(self, config: ForecastOperatorConfig):
@@ -36,14 +36,19 @@ def __init__(self, config: ForecastOperatorConfig):
     def _load_data(self, spec):
         """Loads forecasting input data."""
 
+        loading_start_time = time.time()
         raw_data = utils._load_data(
             filename=spec.historical_data.url,
             format=spec.historical_data.format,
             columns=spec.historical_data.columns,
         )
+        loading_end_time = time.time()
+        logger.info("Loading the data completed in %s seconds", loading_end_time - loading_start_time)
         self.original_user_data = raw_data.copy()
         data_transformer = Transformations(raw_data, spec)
         data = data_transformer.run()
+        transformation_end_time = time.time()
+        logger.info("Transformations are completed in %s seconds", transformation_end_time - loading_end_time)
         try:
             spec.freq = utils.get_frequency_of_datetime(data, spec)
         except TypeError as e:

ads/opctl/operator/lowcode/forecast/model/prophet.py

@@ -378,6 +378,7 @@ def _generate_report(self):
         )
 
     def _custom_predict_prophet(self, data):
+        data[PROPHET_INTERNAL_DATE_COL] = pd.to_datetime(data[PROPHET_INTERNAL_DATE_COL], unit='s')
         return self.models[self.target_columns.index(self.series_id)].predict(
             data.reset_index()
         )["yhat"]

ads/opctl/operator/lowcode/forecast/operator_config.py

@@ -12,7 +12,7 @@
 from ads.opctl.operator.common.utils import _load_yaml_from_uri
 from ads.opctl.operator.common.operator_config import OperatorConfig
 
-from .const import SupportedMetrics
+from .const import SupportedMetrics, SpeedAccuracyMode
 from .const import SupportedModels
 
 @dataclass(repr=True)
@@ -88,6 +88,7 @@ class ForecastOperatorSpec(DataClassSerializable):
     generate_report: bool = None
     generate_metrics: bool = None
     generate_explanations: bool = None
+    explanations_accuracy_mode: str = None
     horizon: int = None
     freq: str = None
     model: str = None
@@ -119,6 +120,7 @@ def __post_init__(self):
             if self.generate_explanations is not None
             else False
         )
+        self.explanations_accuracy_mode = self.explanations_accuracy_mode or SpeedAccuracyMode.FAST_APPROXIMATE
         self.report_theme = self.report_theme or "light"
         self.metrics_filename = self.metrics_filename or "metrics.csv"
         self.test_metrics_filename = self.test_metrics_filename or "test_metrics.csv"

ads/opctl/operator/lowcode/forecast/schema.yaml

@@ -234,6 +234,15 @@ spec:
       meta:
         description: "Explainability, both local and global, can be disabled using this flag. Defaults to false."
 
+    explanations_accuracy_mode:
+      type: string
+      required: false
+      default: FAST_APPROXIMATE
+      allowed:
+        - HIGH_ACCURACY
+        - BALANCED
+        - FAST_APPROXIMATE
+
     generate_report:
       type: boolean
       required: false