enable auto-select-series testcases

Author: codeloop

tests/operators/forecast/test_datasets.py

@@ -2,24 +2,21 @@
 
 # Copyright (c) 2023, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-import os
 import json
-import yaml
-import tempfile
+import os
 import subprocess
+import tempfile
+from copy import deepcopy
+from time import sleep
+
 import pandas as pd
 import pytest
-from time import sleep, time
-from copy import deepcopy
-from pathlib import Path
-import random
-import pathlib
-import datetime
+import yaml
+
 from ads.opctl.operator.cmd import run
 from ads.opctl.operator.lowcode.forecast.__main__ import operate as forecast_operate
 from ads.opctl.operator.lowcode.forecast.operator_config import ForecastOperatorConfig
 
-
 DATASET_PREFIX = f"{os.path.dirname(os.path.abspath(__file__))}/../data/timeseries/"
 
 DATASETS_LIST = [
@@ -37,6 +34,7 @@
     "autots",
     # "lgbforecast",
     "auto-select",
+    "auto-select-series",
 ]
 
 TEMPLATE_YAML = {
@@ -77,14 +75,43 @@
 
 
 def verify_explanations(tmpdirname, additional_cols, target_category_columns):
-    glb_expl = pd.read_csv(f"{tmpdirname}/results/global_explanation.csv", index_col=0)
-    loc_expl = pd.read_csv(f"{tmpdirname}/results/local_explanation.csv")
-    assert loc_expl.shape[0] == PERIODS
-    columns = ["Date", "Series"]
-    if not target_category_columns:
-        columns.remove("Series")
-    for x in columns:
-        assert x in set(loc_expl.columns)
+    result_files = os.listdir(f"{tmpdirname}/results")
+    if model == "auto-select-series":
+        # Find all local and global explanation files
+        local_expl_files = [
+            f
+            for f in result_files
+            if f.startswith("local_explanation_") and f.endswith(".csv")
+        ]
+        global_expl_files = [
+            f
+            for f in result_files
+            if f.startswith("global_explanation_") and f.endswith(".csv")
+        ]
+
+        # Verify for each model's explanation files
+        for loc_file, glb_file in zip(local_expl_files, global_expl_files):
+            glb_expl = pd.read_csv(f"{tmpdirname}/results/{glb_file}", index_col=0)
+            loc_expl = pd.read_csv(f"{tmpdirname}/results/{loc_file}")
+
+            assert loc_expl.shape[0] == PERIODS
+            columns = ["Date", "Series"]
+            if not target_category_columns:
+                columns.remove("Series")
+            for x in columns:
+                assert x in set(loc_expl.columns)
+    else:
+        glb_expl = pd.read_csv(
+            f"{tmpdirname}/results/global_explanation.csv", index_col=0
+        )
+        loc_expl = pd.read_csv(f"{tmpdirname}/results/local_explanation.csv")
+
+        assert loc_expl.shape[0] == PERIODS
+        columns = ["Date", "Series"]
+        if not target_category_columns:
+            columns.remove("Series")
+        for x in columns:
+            assert x in set(loc_expl.columns)
     # for x in additional_cols:
     #     assert x in set(loc_expl.columns)
     #     assert x in set(glb_expl.index)
@@ -159,10 +186,38 @@ def test_load_datasets(model, data_details):
                 target_category_columns=yaml_i["spec"]["target_category_columns"],
             )
         if include_test_data:
-            test_metrics = pd.read_csv(f"{tmpdirname}/results/test_metrics.csv")
-            print(test_metrics)
-            train_metrics = pd.read_csv(f"{tmpdirname}/results/metrics.csv")
-            print(train_metrics)
+            result_files = os.listdir(f"{tmpdirname}/results")
+            if model == "auto-select-series":
+                # Find all metrics files for each model
+                test_metrics_files = [
+                    f
+                    for f in result_files
+                    if f.startswith("test_metrics_") and f.endswith(".csv")
+                ]
+                train_metrics_files = [
+                    f
+                    for f in result_files
+                    if f.startswith("metrics_") and f.endswith(".csv")
+                ]
+
+                # Print metrics for each model
+                for test_file, train_file in zip(
+                    test_metrics_files, train_metrics_files
+                ):
+                    print(
+                        f"\nMetrics for {test_file.replace('test_metrics_', '').replace('.csv', '')}:"
+                    )
+                    test_metrics = pd.read_csv(f"{tmpdirname}/results/{test_file}")
+                    print("Test metrics:")
+                    print(test_metrics)
+                    train_metrics = pd.read_csv(f"{tmpdirname}/results/{train_file}")
+                    print("Train metrics:")
+                    print(train_metrics)
+            else:
+                test_metrics = pd.read_csv(f"{tmpdirname}/results/test_metrics.csv")
+                print(test_metrics)
+                train_metrics = pd.read_csv(f"{tmpdirname}/results/metrics.csv")
+                print(train_metrics)
 
 
 @pytest.mark.parametrize("model", MODELS[:-2])

tests/operators/forecast/test_explainers.py

@@ -2,27 +2,14 @@
 
 # Copyright (c) 2023, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-import datetime
-import json
 import os
-import pathlib
-import random
-import subprocess
 import tempfile
 from copy import deepcopy
-from pathlib import Path
-from time import sleep, time
 
 import numpy as np
-import pandas as pd
 import pytest
-import yaml
 
-from ads.opctl.operator.cmd import run
 from ads.opctl.operator.lowcode.forecast.__main__ import operate as forecast_operate
-from ads.opctl.operator.lowcode.forecast.model.forecast_datasets import (
-    ForecastDatasets,
-)
 from ads.opctl.operator.lowcode.forecast.operator_config import (
     ForecastOperatorConfig,
 )
@@ -33,6 +20,7 @@
     # "automlx", # FIXME: automlx is failing, no errors
     "prophet",
     "neuralprophet",
+    "auto-select-series",
 ]
 
 TEMPLATE_YAML = {
@@ -170,31 +158,31 @@ def test_explanations_output_and_columns(model, freq, num_series):
     global_explanations = results.get_global_explanations()
     local_explanations = results.get_local_explanations()
 
-    assert (
-        not (global_explanations.isna()).all().all()
-    ), "Global explanations contain NaN values"
-    assert (
-        not (global_explanations == 0).all().all()
-    ), "Global explanations contain only 0 values"
-    assert (
-        not (local_explanations.isna()).all().all()
-    ), "Local explanations contain NaN values"
-    assert (
-        not (local_explanations == 0).all().all()
-    ), "Local explanations contain only 0 values"
+    assert not (global_explanations.isna()).all().all(), (
+        "Global explanations contain NaN values"
+    )
+    assert not (global_explanations == 0).all().all(), (
+        "Global explanations contain only 0 values"
+    )
+    assert not (local_explanations.isna()).all().all(), (
+        "Local explanations contain NaN values"
+    )
+    assert not (local_explanations == 0).all().all(), (
+        "Local explanations contain only 0 values"
+    )
 
     additional_columns = list(
         set(additional.columns.tolist())
         - set(operator_config.spec.target_category_columns)
         - {operator_config.spec.datetime_column.name}
     )
     for column in additional_columns:
-        assert (
-            column in global_explanations.T.columns
-        ), f"Column {column} missing in global explanations"
-        assert (
-            column in local_explanations.columns
-        ), f"Column {column} missing in local explanations"
+        assert column in global_explanations.T.columns, (
+            f"Column {column} missing in global explanations"
+        )
+        assert column in local_explanations.columns, (
+            f"Column {column} missing in local explanations"
+        )
 
 
 @pytest.mark.parametrize("model", MODELS)  # MODELS
@@ -221,24 +209,60 @@ def test_explanations_filenames(model, num_series):
         operator_config.spec.local_explanation_filename = local_explanation_filename
 
         results = forecast_operate(operator_config)
-        assert (
-            not results.get_global_explanations().empty
-        ), "Error generating Global Expl"
-        assert not results.get_local_explanations().empty, "Error generating Local Expl"
-
-        global_explanation_path = os.path.join(
-            output_directory, global_explanation_filename
-        )
-        local_explanation_path = os.path.join(
-            output_directory, local_explanation_filename
+        assert not results.get_global_explanations().empty, (
+            "Error generating Global Expl"
         )
+        assert not results.get_local_explanations().empty, "Error generating Local Expl"
 
-        assert os.path.exists(
-            global_explanation_path
-        ), f"Global explanation file not found at {global_explanation_path}"
-        assert os.path.exists(
-            local_explanation_path
-        ), f"Local explanation file not found at {local_explanation_path}"
+        if model == "auto-select-series":
+            # List all files in output directory
+            files = os.listdir(output_directory)
+            # Find all explanation files
+            global_explanation_files = [
+                f
+                for f in files
+                if f.startswith("custom_global_explanation_") and f.endswith(".csv")
+            ]
+            local_explanation_files = [
+                f
+                for f in files
+                if f.startswith("custom_local_explanation_") and f.endswith(".csv")
+            ]
+
+            # Should have at least one file of each type
+            assert len(global_explanation_files) > 0, (
+                "No global explanation files found for auto-select-series"
+            )
+            assert len(local_explanation_files) > 0, (
+                "No local explanation files found for auto-select-series"
+            )
+
+            # Check each file exists
+            for gfile in global_explanation_files:
+                gpath = os.path.join(output_directory, gfile)
+                assert os.path.exists(gpath), (
+                    f"Global explanation file not found at {gpath}"
+                )
+
+            for lfile in local_explanation_files:
+                lpath = os.path.join(output_directory, lfile)
+                assert os.path.exists(lpath), (
+                    f"Local explanation file not found at {lpath}"
+                )
+        else:
+            global_explanation_path = os.path.join(
+                output_directory, global_explanation_filename
+            )
+            local_explanation_path = os.path.join(
+                output_directory, local_explanation_filename
+            )
+
+            assert os.path.exists(global_explanation_path), (
+                f"Global explanation file not found at {global_explanation_path}"
+            )
+            assert os.path.exists(local_explanation_path), (
+                f"Local explanation file not found at {local_explanation_path}"
+            )
 
 
 @pytest.mark.parametrize("model", MODELS)
@@ -297,7 +321,7 @@ def test_explanations_accuracy_mode(mode, model, num_series):
         operator_config.spec.output_directory.url = output_directory
         operator_config.spec.explanations_accuracy_mode = mode
 
-        results = forecast_operate(operator_config)
+        forecast_operate(operator_config)
 
         global_explanation_path = os.path.join(
             output_directory, operator_config.spec.global_explanation_filename
@@ -306,12 +330,12 @@ def test_explanations_accuracy_mode(mode, model, num_series):
             output_directory, operator_config.spec.local_explanation_filename
         )
 
-        assert os.path.exists(
-            global_explanation_path
-        ), f"Global explanation file not found at {global_explanation_path}"
-        assert os.path.exists(
-            local_explanation_path
-        ), f"Local explanation file not found at {local_explanation_path}"
+        assert os.path.exists(global_explanation_path), (
+            f"Global explanation file not found at {global_explanation_path}"
+        )
+        assert os.path.exists(local_explanation_path), (
+            f"Local explanation file not found at {local_explanation_path}"
+        )
 
 
 @pytest.mark.parametrize("model", MODELS)
@@ -345,19 +369,18 @@ def test_explanations_values(model, num_series, freq):
 
         # Check decimal precision for local explanations
         local_numeric = local_explanations.select_dtypes(include=["int64", "float64"])
-        assert np.allclose(local_numeric, np.round(local_numeric, 4), atol=1e-8), \
+        assert np.allclose(local_numeric, np.round(local_numeric, 4), atol=1e-8), (
             "Local explanations have values with more than 4 decimal places"
+        )
 
         # Check decimal precision for global explanations
         global_explanations = results.get_global_explanations()
         global_numeric = global_explanations.select_dtypes(include=["int64", "float64"])
-        assert np.allclose(global_numeric, np.round(global_numeric, 4), atol=1e-8), \
+        assert np.allclose(global_numeric, np.round(global_numeric, 4), atol=1e-8), (
             "Global explanations have values with more than 4 decimal places"
-
-        local_explain_vals = (
-            local_numeric.sum(axis=1)
-            + forecast.fitted_value.mean()
         )
+
+        local_explain_vals = local_numeric.sum(axis=1) + forecast.fitted_value.mean()
         assert np.allclose(
             local_explain_vals,
             forecast[-operator_config.spec.horizon :]["forecast_value"],