Add metrics testing (#87)

* add configs

* add report generator metrics config

* add knn_regr

* add ci testing for knn_regr

* rename algos in cuml

* rename algos in sklearn

* reworked kmeans config for test metrics

* add init to report config

* add opportunity to write path to folder in configs

* pep8

* change readme

* apply comments, part1

* apply comments, part2

* dfs strategy to take jsons from folders

* now you can --configs=folder,.json,folder

* Update README.md

* apply comment

Co-authored-by: Ekaterina Mekhnetsova <mekkatya@gmail.com>

Author: OnlyDeniko

README.md

@@ -74,7 +74,7 @@ Run `python runner.py --configs configs/config_example.json [--output-file resul
 
 Options:
 
-- ``--configs``: specify the path to a configuration file.
+- ``--configs``: specify the path to a configuration file or a folder that contains configuration files.
 - ``--no-intel-optimized``: use Scikit-learn without [Intel(R) Extension for Scikit-learn*](#intelr-extension-for-scikit-learn-support). Now available for [scikit-learn benchmarks](https://github.com/IntelPython/scikit-learn_bench/tree/master/sklearn_bench). By default, the runner uses Intel(R) Extension for Scikit-learn.
 - ``--output-file``: specify the name of the output file for the benchmark result. The default name is `result.json`
 - ``--report``: create an Excel report based on benchmark results. The `openpyxl` library is required.

configs/blogs/skl_2021_3.json

@@ -1,9 +1,9 @@
 {
     "common": {
-        "lib": ["sklearn"],
-        "data-format": ["pandas"],
-        "data-order": ["F"],
-        "dtype": ["float64"]
+        "lib": "sklearn",
+        "data-format": "pandas",
+        "data-order": "F",
+        "dtype": "float64"
     },
     "cases": [
         {
@@ -19,11 +19,11 @@
                     }
                 }
             ],
-            "time-method": ["box_filter"],
-            "time-limit": [50],
-            "n-clusters": [1000],
-            "maxiter": [50],
-            "tol": [0.0]
+            "time-method": "box_filter",
+            "time-limit": 50,
+            "n-clusters": 1000,
+            "maxiter": 50,
+            "tol": 0.0
         },
         {
             "algorithm": "kmeans",
@@ -38,12 +38,12 @@
                     }
                 }
             ],
-            "time-method": ["box_filter"],
-            "time-limit": [50],
-            "n-clusters": [5],
-            "maxiter": [50],
-            "init": ["k-means++"],
-            "tol": [0.0]
+            "time-method": "box_filter",
+            "time-limit": 50,
+            "n-clusters": 5,
+            "maxiter": 50,
+            "init": "k-means++",
+            "tol": 0.0
         },
         {
             "algorithm": "kmeans",
@@ -58,11 +58,11 @@
                     }
                 }
             ],
-            "time-method": ["box_filter"],
-            "time-limit": [50],
-            "n-clusters": [20],
-            "maxiter": [50],
-            "tol": [0.0]
+            "time-method": "box_filter",
+            "time-limit": 50,
+            "n-clusters": 20,
+            "maxiter": 50,
+            "tol": 0.0
         },
         {
             "algorithm": "pca",
@@ -107,12 +107,12 @@
                     }
                 }
             ],
-            "svd-solver": ["full"],
-            "n-components": [10]
+            "svd-solver": "full",
+            "n-components": 10
         },
         {
             "algorithm": "df_clsf",
-            "dtype": ["float32"],
+            "dtype": "float32",
             "dataset": [
                 {
                     "source": "npy",
@@ -129,10 +129,10 @@
                     }
                 }
             ],
-            "num-trees": [50],
-            "max-depth": [16],
-            "max-leaf-nodes": [131072],
-            "max-features": [0.2]
+            "num-trees": 50,
+            "max-depth": 16,
+            "max-leaf-nodes": 131072,
+            "max-features": 0.2
         },
         {
             "algorithm": "ridge",
@@ -146,7 +146,7 @@
                     }
                 }
             ],
-            "alpha": [5]
+            "alpha": 5
         },
         {
             "algorithm": "linear",
@@ -201,8 +201,8 @@
                     }
                 }
             ],
-            "maxiter": [100],
-            "tol": [0]
+            "maxiter": 100,
+            "tol": 0
         },
         {
             "algorithm": "svm",
@@ -222,8 +222,8 @@
                     }
                 }
             ],
-            "C": [500.0],
-            "kernel": ["rbf"]
+            "C": 500.0,
+            "kernel": "rbf"
         },
         {
             "algorithm": "svm",
@@ -243,8 +243,8 @@
                     }
                 }
             ],
-            "C": [1.5e-3],
-            "kernel": ["linear"]
+            "C": 1.5e-3,
+            "kernel": "linear"
         },
         {
             "algorithm": "svm",
@@ -264,8 +264,8 @@
                     }
                 }
             ],
-            "C": [100.0],
-            "kernel": ["linear"]
+            "C": 100.0,
+            "kernel": "linear"
         },
         {
             "algorithm": "svm",
@@ -285,8 +285,8 @@
                     }
                 }
             ],
-            "C": [50.0],
-            "kernel": ["rbf"]
+            "C": 50.0,
+            "kernel": "rbf"
         },
         {
             "algorithm": "nusvc",
@@ -306,8 +306,8 @@
                     }
                 }
             ],
-            "nu": [0.25],
-            "kernel": ["poly"]
+            "nu": 0.25,
+            "kernel": "poly"
         },
         {
             "algorithm": "svr",
@@ -327,8 +327,8 @@
                     }
                 }
             ],
-            "C": [0.1],
-            "kernel": ["poly"]
+            "C": 0.1,
+            "kernel": "poly"
         },
         {
             "algorithm": "nusvr",
@@ -348,9 +348,9 @@
                     }
                 }
             ],
-            "nu": [0.8],
-            "C": [2.0],
-            "kernel": ["rbf"]
+            "nu": 0.8,
+            "C": 2.0,
+            "kernel": "rbf"
         },
         {
             "algorithm": "dbscan",
@@ -386,7 +386,7 @@
         },
         {
             "algorithm": "knn_clsf",
-            "dtype": ["float32"],
+            "dtype": "float32",
             "dataset": [
                 {
                     "source": "synthetic",
@@ -437,11 +437,11 @@
                     }
                 }
             ],
-            "method": ["brute"]
+            "method": "brute"
         },
         {
             "algorithm": "knn_clsf",
-            "dtype": ["float32"],
+            "dtype": "float32",
             "dataset": [
                 {
                     "source": "synthetic",
@@ -468,7 +468,7 @@
                     }
                 }
             ],
-            "method": ["kd_tree"]
+            "method": "kd_tree"
         },
         {
             "algorithm": "train_test_split",
@@ -483,9 +483,9 @@
                     }
                 }
             ],
-            "include-y": [""],
-            "train-size": [0.75],
-            "test-size": [0.25]
+            "include-y": "",
+            "train-size": 0.75,
+            "test-size": 0.25
         }
     ]
 }