ODSC-39815: roc curve bug (#189)

Co-authored-by: Liuda Rudenka <liuda.rudenka@oracle.com>

Author: ahosler

ads/evaluations/statistical_metrics.py

@@ -198,20 +198,27 @@ def _get_binary_metrics(self):
             self.y_true, self.y_pred, pos_label=self.positive_class, average="binary"
         )
 
-        (
-            self.metrics["false_positive_rate"],
-            self.metrics["true_positive_rate"],
-            _,
-        ) = metrics.roc_curve(self.y_true, self.y_pred, pos_label=self.positive_class)
-        self.metrics["auc"] = metrics.auc(
-            self.metrics["false_positive_rate"], self.metrics["true_positive_rate"]
-        )
-
         if self.y_score is not None:
             if not all(0 >= x >= 1 for x in self.y_score):
                 self.y_score = np.asarray(
                     [0 if x < 0 else 1 if x > 1 else x for x in self.y_score]
                 )
+            if len(np.asarray(self.y_score).shape) > 1: 
+                # If the SKLearn classifier doesn't correctly identify the problem as 
+                # binary classification, y_score may be of shape (n_rows, 2) 
+                # instead of (n_rows,)
+                pos_class_idx = self.classes.index(self.positive_class)
+                positive_class_scores = self.y_score[:, pos_class_idx]
+            else:
+                positive_class_scores = self.y_score
+            (
+                self.metrics["false_positive_rate"],
+                self.metrics["true_positive_rate"],
+                _,
+            ) = metrics.roc_curve(y_true=self.y_true, y_score=positive_class_scores, pos_label=self.positive_class)
+            self.metrics["auc"] = metrics.auc(
+                self.metrics["false_positive_rate"], self.metrics["true_positive_rate"]
+            )
             self.y_score = list(self.y_score)
             self.metrics["youden_j"] = (
                 self.metrics["true_positive_rate"] - self.metrics["false_positive_rate"]

tests/unitary/with_extras/evaluator/test_evaluations_evaluator.py

@@ -159,7 +159,7 @@ def test_auc_against_sklearn(self):
         rf_clf = RandomForestClassifier(n_estimators=10, random_state=0).fit(
             X_train, y_train
         )
-        svc_clf = SVC(kernel="linear", C=1.0, random_state=0).fit(X_train, y_train)
+        svc_clf = SVC(kernel="linear", C=1.0, random_state=0, probability=True).fit(X_train, y_train)
 
         bin_lr_model = ADSModel.from_estimator(lr_clf, classes=[0, 1])
         bin_rf_model = ADSModel.from_estimator(rf_clf, classes=[0, 1])
@@ -178,7 +178,8 @@ def test_auc_against_sklearn(self):
             ("rf", bin_rf_model),
             ("svc", svc_model),
         ]:
-            fpr, tpr, _ = roc_curve(test.y, model.est.predict(test.X), pos_label=1)
+            pos_label_idx = model.classes_.index(1)
+            fpr, tpr, _ = roc_curve(test.y, model.est.predict_proba(test.X)[:,pos_label_idx], pos_label=1)
             sklearn_metrics[model_type] = round(auc(fpr, tpr), 4)
 
         assert (