Second iteration of benchmark optimization (#72)

* A huge modelbuilders benchmark renovation + fixing of a bug in a report generator + reporting metrics as well

* preparing for merge

* Applying mypy & pep8 suggestions

* Added metric type as a separate field

* Newlines at the end:)

Author: Igor Rukhovich

.gitignore

@@ -14,5 +14,5 @@ data
 *.npy
 
 # Results
-results.json
+results*.json
 *.xlsx

configs/modelbuilders/lgbm_mb_cpu_config.json

@@ -1,115 +1,109 @@
 {
     "common": {
-        "lib":          "modelbuilders",
-        "data-format":  "pandas",
-        "data-order":   "F",
-        "dtype":        "float32",
-        "algorithm":    "lgbm_mb"
+        "lib": "modelbuilders",
+        "data-format": "pandas",
+        "data-order": "F",
+        "dtype": "float32",
+        "algorithm": "lgbm_mb"
     },
     "cases": [
         {
             "dataset": [
                 {
-                    "source":   "npy",
-                    "name":     "airline-ohe",
-                    "training":
-                    {
-                        "x":    "data/airline-ohe_x_train.npy",
-                        "y":    "data/airline-ohe_y_train.npy"
+                    "source": "npy",
+                    "name": "airline-ohe",
+                    "training": {
+                        "x": "data/airline-ohe_x_train.npy",
+                        "y": "data/airline-ohe_y_train.npy"
                     },
-                    "testing":
-                    {
-                        "x":    "data/airline-ohe_x_test.npy",
-                        "y":    "data/airline-ohe_y_test.npy"
+                    "testing": {
+                        "x": "data/airline-ohe_x_test.npy",
+                        "y": "data/airline-ohe_y_test.npy"
                     }
                 }
             ],
-            "reg-alpha":        0.9,
-            "max-bin":          256,
+            "reg-alpha": 0.9,
+            "max-bin": 256,
             "scale-pos-weight": 2,
-            "learning-rate":    0.1,
-            "subsample":        1,
-            "reg-lambda":       1,
+            "learning-rate": 0.1,
+            "subsample": 1,
+            "reg-lambda": 1,
             "min-child-weight": 0,
-            "max-depth":        8,
-            "max-leaves":       256,
-            "n-estimators":     1000,
-            "objective":        "binary"
+            "max-depth": 8,
+            "max-leaves": 256,
+            "n-estimators": 1000,
+            "objective": "binary"
         },
         {
             "dataset": [
                 {
-                    "source":   "npy",
-                    "name":     "higgs1m",
-                    "training":
-                    {
-                        "x":    "data/higgs1m_x_train.npy",
-                        "y":    "data/higgs1m_y_train.npy"
+                    "source": "npy",
+                    "name": "higgs1m",
+                    "training": {
+                        "x": "data/higgs1m_x_train.npy",
+                        "y": "data/higgs1m_y_train.npy"
                     },
-                    "testing":
-                    {
-                        "x":    "data/higgs1m_x_test.npy",
-                        "y":    "data/higgs1m_y_test.npy"
+                    "testing": {
+                        "x": "data/higgs1m_x_test.npy",
+                        "y": "data/higgs1m_y_test.npy"
                     }
                 }
             ],
-            "reg-alpha":        0.9,
-            "max-bin":          256,
+            "reg-alpha": 0.9,
+            "max-bin": 256,
             "scale-pos-weight": 2,
-            "learning-rate":    0.1,
-            "subsample":        1,
-            "reg-lambda":       1,
+            "learning-rate": 0.1,
+            "subsample": 1,
+            "reg-lambda": 1,
             "min-child-weight": 0,
-            "max-depth":        8,
-            "max-leaves":       256,
-            "n-estimators":     1000,
-            "objective":        "binary"
+            "max-depth": 8,
+            "max-leaves": 256,
+            "n-estimators": 1000,
+            "objective": "binary"
         },
         {
             "dataset": [
                 {
-                    "source":   "npy",
-                    "name":     "mortgage1Q",
-                    "training":
-                    {
-                        "x":    "data/mortgage1Q_x_train.npy",
-                        "y":    "data/mortgage1Q_y_train.npy"
+                    "source": "npy",
+                    "name": "mortgage1Q",
+                    "training": {
+                        "x": "data/mortgage1Q_x_train.npy",
+                        "y": "data/mortgage1Q_y_train.npy"
                     }
                 }
             ],
-            "n-estimators":     100,
-            "objective":        "regression",
-            "max-depth":        8,
+            "n-estimators": 100,
+            "objective": "regression",
+            "max-depth": 8,
             "scale-pos-weight": 2,
-            "learning-rate":    0.1,
-            "subsample":        1,
-            "reg-alpha":        0.9,
-            "reg-lambda":       1,
+            "learning-rate": 0.1,
+            "subsample": 1,
+            "reg-alpha": 0.9,
+            "reg-lambda": 1,
             "min-child-weight": 0,
-            "max-leaves":       256
+            "max-leaves": 256
         },
         {
             "dataset": [
                 {
-                    "source":   "npy",
-                    "name":     "mlsr",
-                    "training":
-                    {
-                        "x":    "data/mlsr_x_train.npy",
-                        "y":    "data/mlsr_y_train.npy"
+                    "source": "npy",
+                    "name": "mlsr",
+                    "training": {
+                        "x": "data/mlsr_x_train.npy",
+                        "y": "data/mlsr_y_train.npy"
                     }
                 }
             ],
-            "max-bin":          256,
-            "learning-rate":    0.3,
-            "subsample":        1,
-            "reg-lambda":       2,
+            "max-bin": 256,
+            "learning-rate": 0.3,
+            "subsample": 1,
+            "reg-lambda": 2,
             "min-child-weight": 1,
-            "min-split-loss":   0.1,
-            "max-depth":        8,
-            "max-leaves":       256,
-            "n-estimators":     200,
-            "objective":        "multiclass"
+            "min-split-loss": 0.1,
+            "max-depth": 8,
+            "max-leaves": 256,
+            "n-estimators": 200,
+            "objective": "multiclass"
         }
     ]
 }