Include estimator class in results files and better testing for get_estimator methods (#331)

* better testing for get_estimator bugs

* xgboost

* mypy

* mypy

* mypy

Author: MatthewMiddlehurst

pyproject.toml

@@ -152,8 +152,9 @@ per-file-ignores = [
 [tool.mypy]
 mypy_path = "tsml_eval/"
 ignore_missing_imports = true
+follow_imports = "silent"
 exclude = [
-    "_wip",
+    "_wip/",
     # Ignore the publications symlinks and its contents
     "tsml_eval/publications/2023",
 ]

tsml_eval/experiments/_get_clusterer.py

@@ -30,7 +30,7 @@
 distance_based_clusterers = [
     "kmeans-euclidean",
     "kmeans-squared",
-    "kmeans-dtw",
+    ["kmeans-dtw", "timeserieskmeans"],
     "kmeans-ddtw",
     "kmeans-wdtw",
     "kmeans-wddtw",
@@ -43,7 +43,7 @@
     "kmeans-shape_dtw",
     "kmedoids-euclidean",
     "kmedoids-squared",
-    "kmedoids-dtw",
+    ["kmedoids-dtw", "timeserieskmedoids"],
     "kmedoids-ddtw",
     "kmedoids-wdtw",
     "kmedoids-wddtw",
@@ -56,7 +56,7 @@
     "kmedoids-shape_dtw",
     "clarans-euclidean",
     "clarans-squared",
-    "clarans-dtw",
+    ["clarans-dtw", "timeseriesclarans"],
     "clarans-ddtw",
     "clarans-wdtw",
     "clarans-wddtw",
@@ -69,7 +69,7 @@
     "clarans-shape_dtw",
     "clara-euclidean",
     "clara-squared",
-    "clara-dtw",
+    ["clara-dtw", "timeseriesclara"],
     "clara-ddtw",
     "clara-wdtw",
     "clara-wddtw",
@@ -114,7 +114,7 @@
     "kmeans-ssg-ba-msm",
     "kmeans-ssg-ba-adtw",
     "kmeans-ssg-ba-shape_dtw",
-    "som-dtw",
+    ["som-dtw", "elasticsom"],
     "som-ddtw",
     "som-wdtw",
     "som-wddtw",
@@ -126,16 +126,8 @@
     "som-adtw",
     "som-shape_dtw",
     "som-soft_dtw",
-    "ksc",
-    "kshape",
-    "timeserieskmeans",
-    "timeserieskmedoids",
-    "timeseriesclarans",
-    "timeseriesclara",
-    "elasticsom",
-    "kspectralcentroid",
-    "timeserieskshape",
-    "timeserieskernelkmeans",
+    ["kspectralcentroid", "ksc"],
+    ["timeserieskshape", "kshape"],
 ]
 feature_based_clusterers = [
     ["catch22", "catch22clusterer"],
@@ -423,7 +415,7 @@ def _set_clusterer_distance_based(
             random_state=random_state,
             **kwargs,
         )
-    elif "kshape" in c or "timeserieskshape" in c:
+    elif "kshape" in c:
         return TimeSeriesKShape(
             init=init_algorithm,
             max_iter=50,
@@ -432,7 +424,7 @@ def _set_clusterer_distance_based(
             random_state=random_state,
             **kwargs,
         )
-    elif c == "timeserieskernelkmeans" or c == "kernelkmeans":
+    elif "timeserieskernelkmeans" in c:
         return TimeSeriesKernelKMeans(
             max_iter=50,
             n_init=10,

tsml_eval/experiments/experiments.py

@@ -230,6 +230,9 @@ def run_classification_experiment(
             dataset_name,
             results_path,
             full_path=False,
+            first_line_classifier_name=(
+                f"{classifier_name} ({type(classifier).__name__})"
+            ),
             split="TRAIN",
             resample_id=resample_id,
             time_unit="MILLISECONDS",
@@ -280,6 +283,9 @@ def run_classification_experiment(
             dataset_name,
             results_path,
             full_path=False,
+            first_line_classifier_name=(
+                f"{classifier_name} ({type(classifier).__name__})"
+            ),
             split="TEST",
             resample_id=resample_id,
             time_unit="MILLISECONDS",
@@ -552,6 +558,7 @@ def run_regression_experiment(
             dataset_name,
             results_path,
             full_path=False,
+            first_line_regressor_name=f"{regressor_name} ({type(regressor).__name__})",
             split="TRAIN",
             resample_id=resample_id,
             time_unit="MILLISECONDS",
@@ -597,6 +604,7 @@ def run_regression_experiment(
             dataset_name,
             results_path,
             full_path=False,
+            first_line_regressor_name=f"{regressor_name} ({type(regressor).__name__})",
             split="TEST",
             resample_id=resample_id,
             time_unit="MILLISECONDS",
@@ -916,6 +924,7 @@ def run_clustering_experiment(
             dataset_name,
             results_path,
             full_path=False,
+            first_line_clusterer_name=f"{clusterer_name} ({type(clusterer).__name__})",
             split="TRAIN",
             resample_id=resample_id,
             time_unit="MILLISECONDS",
@@ -960,6 +969,7 @@ def run_clustering_experiment(
             dataset_name,
             results_path,
             full_path=False,
+            first_line_clusterer_name=f"{clusterer_name} ({type(clusterer).__name__})",
             split="TEST",
             resample_id=resample_id,
             time_unit="MILLISECONDS",
@@ -1197,6 +1207,7 @@ def run_forecasting_experiment(
         dataset_name,
         results_path,
         full_path=False,
+        first_line_forecaster_name=f"{forecaster_name} ({type(forecaster).__name__})",
         split="TEST",
         random_seed=random_seed,
         time_unit="MILLISECONDS",

tsml_eval/experiments/tests/test_classification.py

@@ -171,7 +171,7 @@ def test_run_classification_experiment_invalid_estimator():
 
 def test_get_classifier_by_name():
     """Test get_classifier_by_name method."""
-    classifier_lists = [
+    classifier_name_lists = [
         _get_classifier.convolution_based_classifiers,
         _get_classifier.deep_learning_classifiers,
         _get_classifier.dictionary_based_classifiers,
@@ -184,12 +184,14 @@ def test_get_classifier_by_name():
         _get_classifier.vector_classifiers,
     ]
 
+    # filled by _check_set_method
+    classifier_list = []
     classifier_dict = {}
     all_classifier_names = []
-
-    for classifier_list in classifier_lists:
+    for classifier_name_list in classifier_name_lists:
         _check_set_method(
             get_classifier_by_name,
+            classifier_name_list,
             classifier_list,
             classifier_dict,
             all_classifier_names,

tsml_eval/experiments/tests/test_clustering.py

@@ -173,7 +173,7 @@ def test_run_clustering_experiment_invalid_estimator():
 
 def test_get_clusterer_by_name():
     """Test get_clusterer_by_name method."""
-    clusterer_lists = [
+    clusterer_name_lists = [
         _get_clusterer.deep_learning_clusterers,
         _get_clusterer.distance_based_clusterers,
         _get_clusterer.feature_based_clusterers,
@@ -187,29 +187,30 @@ def test_get_clusterer_by_name():
         "base_estimator",
     ]
 
+    clusterer_list = []
     clusterer_dict = {}
     all_clusterer_names = []
-
-    for clusterer_list in clusterer_lists:
-        estimatorrs = _check_set_method(
+    for clusterer_name_list in clusterer_name_lists:
+        _check_set_method(
             get_clusterer_by_name,
+            clusterer_name_list,
             clusterer_list,
             clusterer_dict,
             all_clusterer_names,
-            return_estimator=True,
         )
 
-        # Check that clusterers with estimator parameters which are likely to be
-        # a sub-estimator are not None so n_clusters can be set
-        for clusterer in estimatorrs:
+    # Check that clusterers with parameters which are likely to be
+    # a sub-estimator are not None so n_clusters can be set
+    for clusterers in clusterer_list:
+        for c in clusterers:
             for param_name in clusterer_non_default_params:
-                params = clusterer.get_params()
+                params = c.get_params()
                 if param_name in params:
                     assert params[param_name] is not None, (
                         f"Clusterers which have an estimator parameter i.e. "
                         f"pipelines and deep learners must not have None as the "
                         f"estimator. Found None for {param_name} in "
-                        f"{clusterer.__class__.__name__}"
+                        f"{c.__class__.__name__}"
                     )
 
     _check_set_method_results(
@@ -230,6 +231,7 @@ def test_aeon_clusterers_available():
         "ClustererPipeline",
         "SklearnClustererWrapper",
         # just missing
+        "TimeSeriesKernelKMeans",
     ]
 
     est = [e for e, _ in all_estimators(type_filter="clusterer")]

tsml_eval/experiments/tests/test_data_transform.py

@@ -9,14 +9,15 @@
 
 def test_get_data_transform_by_name():
     """Test get_data_transform_by_name method."""
-    transform_lists = [_get_data_transform.transformers]
+    transform_name_lists = [_get_data_transform.transformers]
 
+    transform_list = []
     transform_dict = {}
     all_transform_names = []
-
-    for transform_list in transform_lists:
+    for transform_name_list in transform_name_lists:
         _check_set_method(
             get_data_transform_by_name,
+            transform_name_list,
             transform_list,
             transform_dict,
             all_transform_names,

tsml_eval/experiments/tests/test_regression.py

@@ -170,7 +170,7 @@ def test_run_regression_experiment_invalid_estimator():
 
 def test_get_regressor_by_name():
     """Test get_regressor_by_name method."""
-    regressor_lists = [
+    regressor_name_lists = [
         _get_regressor.convolution_based_regressors,
         _get_regressor.deep_learning_regressors,
         _get_regressor.distance_based_regressors,
@@ -182,12 +182,13 @@ def test_get_regressor_by_name():
         _get_regressor.vector_regressors,
     ]
 
+    regressor_list = []
     regressor_dict = {}
     all_regressor_names = []
-
-    for regressor_list in regressor_lists:
+    for regressor_name_list in regressor_name_lists:
         _check_set_method(
             get_regressor_by_name,
+            regressor_name_list,
             regressor_list,
             regressor_dict,
             all_regressor_names,

tsml_eval/publications/y2023/distance_based_clustering/tests/test_set_distance_clusterer.py

@@ -11,12 +11,13 @@
 
 def test_set_distance_clusterer():
     """Test set_distance_clusterer method."""
+    clusterer_list = []
     clusterer_dict = {}
     all_clusterer_names = []
-
     _check_set_method(
         _set_distance_clusterer,
         distance_based_clusterers,
+        clusterer_list,
         clusterer_dict,
         all_clusterer_names,
     )

tsml_eval/publications/y2023/rist_pipeline/tests/test_set_estimator.py

@@ -13,12 +13,13 @@
 
 def test_set_rist_classifier():
     """Test set_rist_classifier method."""
+    classifier_list = []
     classifier_dict = {}
     all_classifier_names = []
-
     _check_set_method(
         _set_rist_classifier,
         rist_classifiers,
+        classifier_list,
         classifier_dict,
         all_classifier_names,
     )
@@ -38,12 +39,13 @@ def test_set_rist_classifier_invalid():
 
 def test_set_rist_regressor():
     """Test set_rist_regressors method."""
+    regressor_list = []
     regressor_dict = {}
     all_regressor_names = []
-
     _check_set_method(
         _set_rist_regressor,
         rist_regressors,
+        regressor_list,
         regressor_dict,
         all_regressor_names,
     )

tsml_eval/publications/y2023/tsc_bakeoff/tests/test_set_classifier.py

@@ -11,12 +11,13 @@
 
 def test_set_bakeoff_classifier():
     """Test set_bakeoff_classifier method."""
+    classifier_list = []
     classifier_dict = {}
     all_classifier_names = []
-
     _check_set_method(
         _set_bakeoff_classifier,
         bakeoff_classifiers,
+        classifier_list,
         classifier_dict,
         all_classifier_names,
     )