Merge pull request #24 from RonasIT/vglinskii/more-metrics

Author: dmitryusenko

README.md

@@ -72,18 +72,25 @@ We have a list of built-in metric calculators for popular metrics:
 - PrecisionMetricCalculator
 - RecallMetricCalculator
 - SpecificityMetricCalculator
-- F1ScoreMetricCalculator
 - FNRMetricCalculator
 - FPRMetricCalculator
+- NPVMetricCalculator
+- MCCMetricCalculator
+- FBetaScoreMetricCalculator
+- ROCAUCMetricCalculator
+- PRAUCMetricCalculator
+- BrierLossMetricCalculator
+- BinaryCrossentropyMetricCalculator
+- CohenKappaMetricCalculator
 
 You can implement your own `MetricCalculator`. In the example below, we define
 a metric calculator for `precision`. For that we create a `PrecisionMetricCalculator`
 class extending the `MetricCalculator` base class provided by the library and
 implementing `calculate` method.
 
 ```typescript
-class PrecisionMetricCalculator extends MetricCalculator {
-  public calculate(trueValues: Float32Array, predictedValues: Float32Array): Metric {
+export class PrecisionMetricCalculator extends MetricCalculator {
+  public calculate({ trueValues, predictedValues }: TestingResult): Metric {
     const { tp, fp } = new ConfusionMatrix(trueValues, predictedValues);
 
     return new Metric({
@@ -258,7 +265,7 @@ const ownsTheCar = await classifier.predict([0.2, 0.76, 0]);
 - [x] Asynchronously loaded datasets ([#14](https://github.com/RonasIT/tfjs-node-helpers/issues/14))
 - [x] Feature normalization ([#5](https://github.com/RonasIT/tfjs-node-helpers/issues/5))
 - [x] Custom metrics ([#18](https://github.com/RonasIT/tfjs-node-helpers/issues/18))
-- [ ] Add more metrics ([#17](https://github.com/RonasIT/tfjs-node-helpers/issues/17))
+- [x] Add more metrics ([#17](https://github.com/RonasIT/tfjs-node-helpers/issues/17))
 - [ ] Refactor features ([#25](https://github.com/RonasIT/tfjs-node-helpers/issues/25))
 - [ ] Task-oriented architecture ([#26](https://github.com/RonasIT/tfjs-node-helpers/issues/26))
 - [ ] Categorical features ([#19](https://github.com/RonasIT/tfjs-node-helpers/issues/19))

packages/tfjs-node-helpers-example/src/app/app.ts

@@ -2,13 +2,20 @@ import {
   AccuracyMetricCalculator,
   BinaryClassificationTrainer,
   BinaryClassifier,
-  F1ScoreMetricCalculator,
+  BrierLossMetricCalculator,
   FNRMetricCalculator,
   FPRMetricCalculator,
+  BinaryCrossentropyMetricCalculator,
   makeChunkedDataset,
+  PRAUCMetricCalculator,
   PrecisionMetricCalculator,
   RecallMetricCalculator,
-  SpecificityMetricCalculator
+  ROCAUCMetricCalculator,
+  SpecificityMetricCalculator,
+  CohenKappaMetricCalculator,
+  NPVMetricCalculator,
+  MCCMetricCalculator,
+  FBetaScoreMetricCalculator
 } from '@ronas-it/tfjs-node-helpers';
 import { data, layers, TensorContainer } from '@tensorflow/tfjs-node';
 import { AgeFeatureExtractor } from './feature-extractors/age';
@@ -34,18 +41,23 @@ async function train(): Promise<void> {
       new GenderFeatureExtractor()
     ],
     outputFeatureExtractor: new OwnsTheCarFeatureExtractor(),
-    inputFeatureNormalizers: [
-      new AgeMinMaxFeatureNormalizer(),
-      new AnnualSalaryMinMaxFeatureNormalizer()
-    ],
+    inputFeatureNormalizers: [new AgeMinMaxFeatureNormalizer(), new AnnualSalaryMinMaxFeatureNormalizer()],
     metricCalculators: [
       new AccuracyMetricCalculator(),
       new PrecisionMetricCalculator(),
-      new F1ScoreMetricCalculator(),
+      new FBetaScoreMetricCalculator(1),
       new SpecificityMetricCalculator(),
       new RecallMetricCalculator(),
       new FNRMetricCalculator(),
-      new FPRMetricCalculator()
+      new FPRMetricCalculator(),
+      new NPVMetricCalculator(),
+      new MCCMetricCalculator(),
+      new FBetaScoreMetricCalculator(2),
+      new ROCAUCMetricCalculator(),
+      new PRAUCMetricCalculator(),
+      new BrierLossMetricCalculator(),
+      new BinaryCrossentropyMetricCalculator(),
+      new CohenKappaMetricCalculator()
     ]
   });
 

packages/tfjs-node-helpers/src/classification/binary-classification-trainer.ts

@@ -189,13 +189,13 @@ export class BinaryClassificationTrainer {
     const predictions = this.model.predict(testXs) as Tensor;
     const binarizedPredictions = binarize(predictions);
 
-    const trueValues = await testYs.data<'float32'>();
-    const predictedValues = await binarizedPredictions.data<'float32'>();
-
-    const confusionMatrix = new ConfusionMatrix(trueValues, predictedValues);
+    const confusionMatrix = new ConfusionMatrix(testYs, binarizedPredictions);
     const metrics = calculateMetrics({
-      trueValues,
-      predictedValues,
+      testingResult: {
+        trueValues: testYs,
+        predictedValues: binarizedPredictions,
+        probabilities: predictions
+      },
       metricCalculators: this.metricCalculators
     });
 

packages/tfjs-node-helpers/src/index.ts

@@ -13,11 +13,19 @@ export * from './utils/make-chunked-dataset';
 export * from './testing/calculate-metrics';
 export * from './testing/confusion-matrix';
 export * from './testing/metric';
+export * from './testing/result';
 export * from './testing/metric-calculator';
 export * from './testing/metrics/accuracy';
 export * from './testing/metrics/precision';
-export * from './testing/metrics/f1-score';
 export * from './testing/metrics/recall';
 export * from './testing/metrics/specificity';
 export * from './testing/metrics/fpr';
 export * from './testing/metrics/fnr';
+export * from './testing/metrics/fbeta-score';
+export * from './testing/metrics/mcc';
+export * from './testing/metrics/npv';
+export * from './testing/metrics/roc-auc';
+export * from './testing/metrics/pr-auc';
+export * from './testing/metrics/brier-loss';
+export * from './testing/metrics/binary-crossentropy';
+export * from './testing/metrics/cohen-kappa';

packages/tfjs-node-helpers/src/testing/calculate-metrics.ts

@@ -1,12 +1,11 @@
 import { Metric } from './metric';
 import { MetricCalculator } from './metric-calculator';
+import { TestingResult } from './result';
 
 export const calculateMetrics = ({
-  trueValues,
-  predictedValues,
+  testingResult,
   metricCalculators
 }: {
-  trueValues: Float32Array;
-  predictedValues: Float32Array;
+  testingResult: TestingResult;
   metricCalculators: Array<MetricCalculator>;
-}): Array<Metric> => metricCalculators.map((calculator) => calculator.calculate(trueValues, predictedValues));
+}): Array<Metric> => metricCalculators.map((calculator) => calculator.calculate(testingResult));

packages/tfjs-node-helpers/src/testing/confusion-matrix.ts

@@ -1,28 +1,28 @@
+import { math, Tensor } from '@tensorflow/tfjs-node';
+
 export class ConfusionMatrix {
   public tp: number;
   public tn: number;
   public fp: number;
   public fn: number;
 
-  constructor(trueValues: Float32Array, predictedValues: Float32Array) {
-    this.tp = 0;
-    this.tn = 0;
-    this.fp = 0;
-    this.fn = 0;
+  constructor(trueValues: Tensor, predictedValues: Tensor) {
+    const trueY = trueValues.as1D();
+    const predY = predictedValues.as1D();
+    const numberOfClasses = 2;
+
+    const confusionMatrix = math.confusionMatrix(trueY, predY, numberOfClasses);
+
+    trueY.dispose();
+    predY.dispose();
+
+    const [[tn, fp], [fn, tp]] = confusionMatrix.arraySync();
 
-    for (let index = 0; index < trueValues.length; index++) {
-      const trueValue = trueValues[index];
-      const predictedValue = predictedValues[index];
+    confusionMatrix.dispose();
 
-      if (trueValue === 1 && predictedValue === 1) {
-        this.tp++;
-      } else if (trueValue === 0 && predictedValue === 0) {
-        this.tn++;
-      } else if (trueValue === 0 && predictedValue === 1) {
-        this.fp++;
-      } else {
-        this.fn++;
-      }
-    }
+    this.tp = tp;
+    this.fp = fp;
+    this.fn = fn;
+    this.tn = tn;
   }
 }

packages/tfjs-node-helpers/src/testing/metric-calculator.ts

@@ -1,5 +1,6 @@
 import { Metric } from './metric';
+import { TestingResult } from './result';
 
 export abstract class MetricCalculator {
-  public abstract calculate(trueValues: Float32Array, predictedValues: Float32Array): Metric;
+  public abstract calculate(testingResult: TestingResult): Metric;
 }

packages/tfjs-node-helpers/src/testing/metrics/accuracy.ts

@@ -1,14 +1,24 @@
-import { ConfusionMatrix } from '../confusion-matrix';
+import { metrics } from '@tensorflow/tfjs-node';
 import { Metric } from '../metric';
 import { MetricCalculator } from '../metric-calculator';
+import { TestingResult } from '../result';
 
 export class AccuracyMetricCalculator extends MetricCalculator {
-  public calculate(trueValues: Float32Array, predictedValues: Float32Array): Metric {
-    const { tp, tn, fp, fn } = new ConfusionMatrix(trueValues, predictedValues);
+  public calculate({ trueValues, predictedValues }: TestingResult): Metric {
+    const trueY = trueValues.as1D();
+    const predY = predictedValues.as1D();
+    const valueTensor = metrics.binaryAccuracy(trueY, predY);
+
+    trueY.dispose();
+    predY.dispose();
+
+    const [value] = valueTensor.dataSync();
+
+    valueTensor.dispose();
 
     return new Metric({
       title: 'Accuracy',
-      value: (tp + tn) / (tp + tn + fp + fn)
+      value
     });
   }
 }

packages/tfjs-node-helpers/src/testing/metrics/binary-crossentropy.ts

@@ -0,0 +1,24 @@
+import { metrics } from '@tensorflow/tfjs-node';
+import { Metric } from '../metric';
+import { MetricCalculator } from '../metric-calculator';
+import { TestingResult } from '../result';
+
+export class BinaryCrossentropyMetricCalculator extends MetricCalculator {
+  public calculate({ trueValues, probabilities }: TestingResult): Metric {
+    const trueY = trueValues.as1D();
+    const predY = probabilities.as1D();
+    const valueTensor = metrics.binaryCrossentropy(trueY, predY);
+
+    trueY.dispose();
+    predY.dispose();
+
+    const [value] = valueTensor.dataSync();
+
+    valueTensor.dispose();
+
+    return new Metric({
+      title: 'Binary Cross-entropy',
+      value
+    });
+  }
+}

packages/tfjs-node-helpers/src/testing/metrics/brier-loss.ts

@@ -0,0 +1,24 @@
+import { metrics } from '@tensorflow/tfjs-node';
+import { Metric } from '../metric';
+import { MetricCalculator } from '../metric-calculator';
+import { TestingResult } from '../result';
+
+export class BrierLossMetricCalculator extends MetricCalculator {
+  public calculate({ trueValues, probabilities }: TestingResult): Metric {
+    const trueY = trueValues.as1D();
+    const predY = probabilities.as1D();
+    const valueTensor = metrics.MSE(trueY, predY);
+
+    trueY.dispose();
+    predY.dispose();
+
+    const [value] = valueTensor.dataSync();
+
+    valueTensor.dispose();
+
+    return new Metric({
+      title: 'Brier Loss',
+      value
+    });
+  }
+}