changes for error in explanation

Author: govarsha

ads/opctl/operator/lowcode/forecast/model/arima.py

@@ -245,6 +245,9 @@ def _custom_predict(
             """
             data: ForecastDatasets.get_data_at_series(s_id)
             """
+            if series_id in self.constant_cols:
+                data = data.drop(columns=self.constant_cols[series_id])
+
             data = data.drop([target_col], axis=1)
             data[dt_column_name] = seconds_to_datetime(
                 data[dt_column_name], dt_format=self.spec.datetime_column.format

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -22,6 +22,7 @@
     seconds_to_datetime,
     datetime_to_seconds,
 )
+from ads.opctl.operator.lowcode.forecast.utils import _label_encode_dataframe
 
 AUTOMLX_N_ALGOS_TUNED = 4
 AUTOMLX_DEFAULT_SCORE_METRIC = "neg_sym_mean_abs_percent_error"
@@ -51,8 +52,13 @@ def set_kwargs(self):
         ] = self.spec.preprocessing or model_kwargs_cleaned.get("preprocessing", True)
         return model_kwargs_cleaned, time_budget
 
-    def preprocess(self, data, series_id=None):
-        return data.set_index(self.spec.datetime_column.name)
+
+    def preprocess(self, data, series_id=None):  # TODO: re-use self.le for explanations
+        _, df_encoded = _label_encode_dataframe(
+            data,
+            no_encode={self.spec.datetime_column.name, self.original_target_column},
+        )
+        return df_encoded.set_index(self.spec.datetime_column.name)
 
     @runtime_dependency(
         module="automlx",

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -88,7 +88,7 @@ def __init__(self, config: ForecastOperatorConfig, datasets: ForecastDatasets):
         self.formatted_local_explanation = None
 
         self.forecast_col_name = "yhat"
-        self.perform_tuning = self.spec.tuning != None
+        self.perform_tuning = (self.spec.tuning != None) and (self.spec.tuning.n_trials != None)
 
     def generate_report(self):
         """Generates the forecasting report."""
@@ -657,20 +657,18 @@ def explain_model(self):
             if s_id in self.models:
 
                 explain_predict_fn = self.get_explain_predict_fn(series_id=s_id)
-                if self.spec.model == SupportedModels.Arima and s_id in self.constant_cols:
-                    data_i = data_i.drop(columns=self.constant_cols[s_id])
                 data_trimmed = data_i.tail(max(int(len(data_i) * ratio), 5)).reset_index(
                     drop=True
                 )
                 data_trimmed[datetime_col_name] = data_trimmed[datetime_col_name].apply(
                     lambda x: x.timestamp()
                 )
 
-                # Explainer fails when boolean columns are passed for arima
-                if self.spec.model == SupportedModels.Arima:
-                    _, data_trimmed_encoded = _label_encode_dataframe(
-                        data_trimmed, no_encode={datetime_col_name, self.original_target_column}
-                    )
+                # Explainer fails when boolean columns are passed
+
+                _, data_trimmed_encoded = _label_encode_dataframe(
+                    data_trimmed, no_encode={datetime_col_name, self.original_target_column}
+                )
 
                 kernel_explnr = PermutationExplainer(
                     model=explain_predict_fn, masker=data_trimmed_encoded
@@ -716,16 +714,13 @@ def local_explainer(self, kernel_explainer, series_id, datetime_col_name) -> Non
         """
         data = self.datasets.get_horizon_at_series(s_id=series_id)
         # columns that were dropped in train_model in arima, should be dropped here as well
-        if self.spec.model == SupportedModels.Arima and series_id in self.constant_cols:
-            data = data.drop(columns=self.constant_cols[series_id])
         data[datetime_col_name] = datetime_to_seconds(data[datetime_col_name])
         data = data.reset_index(drop=True)
 
-        # Explainer fails when boolean columns are passed for arima
-        if self.spec.model == SupportedModels.Arima:
-            _, data = _label_encode_dataframe(
-                data, no_encode={datetime_col_name, self.original_target_column}
-            )
+        # Explainer fails when boolean columns are passed
+        _, data = _label_encode_dataframe(
+            data, no_encode={datetime_col_name, self.original_target_column}
+        )
         # Generate local SHAP values using the kernel explainer
         local_kernel_explnr_vals = kernel_explainer.shap_values(data)
 

tests/operators/forecast/test_errors.py

@@ -524,7 +524,7 @@ def test_all_series_failure(model):
     module_to_patch = {
         "arima": 'pmdarima.auto_arima',
         "autots": 'autots.AutoTS',
-        "automlx": 'automl.Pipeline',
+        "automlx": 'automlx.Pipeline',
         "prophet": 'prophet.Prophet',
         "neuralprophet": 'neuralprophet.NeuralProphet'
     }
@@ -551,7 +551,7 @@ def test_all_series_failure(model):
             local_fn = f"{tmpdirname}/results/local_explanation.csv"
             assert os.path.exists(local_fn), f"Local explanation file not found at {report_path}"
 
-@pytest.mark.parametrize("model", ["arima", "automlx"])
+@pytest.mark.parametrize("model", MODELS)
 def test_arima_automlx_errors(operator_setup, model):
     tmpdirname = operator_setup
     historical_data_path, additional_data_path = setup_faulty_rossman()
@@ -572,14 +572,15 @@ def test_arima_automlx_errors(operator_setup, model):
     outputs get generated and that error is shown in errors.json
     """
 
+    """
+    explanations generation is failing when boolean columns are passed. So we added label_encode before passing data to
+     explainer
+    """
+
     yaml_i['spec']['horizon'] = 10
     yaml_i['spec']['preprocessing'] = True
     yaml_i['spec']['generate_explanations'] = True
     yaml_i['spec']['model'] = model
-    if model == "automlx":
-        yaml_i['spec']['model_kwargs'] = {
-            'model_list': ['ProphetForecaster']
-        }
 
     run_yaml(tmpdirname=tmpdirname, yaml_i=yaml_i, output_data_path=output_data_path, test_metrics_check=False)
 
@@ -594,23 +595,24 @@ def test_arima_automlx_errors(operator_setup, model):
     error_path = f"{tmpdirname}/results/errors.json"
     if model == "arima":
         assert not os.path.exists(error_path), f"Error file not found at {error_path}"
-    else:
+    elif model == "automlx":
         assert os.path.exists(error_path), f"Error file not found at {error_path}"
         with open(error_path, 'r') as error_file:
             error_content = json.load(error_file)
             assert "Input data does not have a consistent (in terms of diff) DatetimeIndex." in error_content["13"][
                 "error"], "Error message mismatch"
 
-    global_fn = f"{tmpdirname}/results/global_explanation.csv"
-    assert os.path.exists(global_fn), f"Global explanation file not found at {report_path}"
+    if model != "autots":
+        global_fn = f"{tmpdirname}/results/global_explanation.csv"
+        assert os.path.exists(global_fn), f"Global explanation file not found at {report_path}"
 
-    local_fn = f"{tmpdirname}/results/local_explanation.csv"
-    assert os.path.exists(local_fn), f"Local explanation file not found at {report_path}"
+        local_fn = f"{tmpdirname}/results/local_explanation.csv"
+        assert os.path.exists(local_fn), f"Local explanation file not found at {report_path}"
 
-    glb_expl = pd.read_csv(global_fn, index_col=0)
-    loc_expl = pd.read_csv(local_fn)
-    assert not glb_expl.empty
-    assert not loc_expl.empty
+        glb_expl = pd.read_csv(global_fn, index_col=0)
+        loc_expl = pd.read_csv(local_fn)
+        assert not glb_expl.empty
+        assert not loc_expl.empty
 
 
 def test_smape_error():
@@ -631,11 +633,7 @@ def test_date_format(operator_setup, model):
     yaml_i["spec"]["model"] = model
     if model == "autots":
         yaml_i["spec"]["model_kwargs"] = {"model_list": "superfast"}
-    if model == "automlx":
-        yaml_i['spec']['model_kwargs'] = {
-            'model_list': ['ProphetForecaster'],
-            "time_budget": 1
-        }
+
     run_yaml(tmpdirname=tmpdirname, yaml_i=yaml_i, output_data_path=output_data_path, test_metrics_check=False)
     assert pd.read_csv(additional_data_path)['Date'].equals(pd.read_csv(f"{tmpdirname}/results/forecast.csv")['Date'])