Merge pull request #22 from RonasIT/vglinskii/custom-metrics

Author: dmitryusenko

.eslintrc.json

@@ -24,7 +24,10 @@
     {
       "files": ["*.ts", "*.tsx"],
       "extends": ["plugin:@nrwl/nx/typescript"],
-      "rules": {}
+      "rules": {
+        "@typescript-eslint/no-inferrable-types": "off",
+        "semi": "error"
+      }
     },
     {
       "files": ["*.js", "*.jsx"],

.prettierrc

@@ -1,3 +1,5 @@
 {
-  "singleQuote": true
+  "singleQuote": true,
+  "trailingComma": "none",
+  "printWidth": 120
 }

README.md

@@ -60,6 +60,40 @@ class GenderFeatureExtractor extends FeatureExtractor<DatasetItem, FeatureType>
 That's it! Now we can use the defined feature extractor to extract valuable
 information from our dataset.
 
+### Metrics
+
+After your model has been trained it's important to evaluate it.
+One way to do this is by analyzing *metrics*.
+The library helps measure model performance by passing a list of
+metric calculators to the model trainer.
+
+We have a list of built-in metric calculators for popular metrics:
+- AccuracyMetricCalculator
+- PrecisionMetricCalculator
+- RecallMetricCalculator
+- SpecificityMetricCalculator
+- F1ScoreMetricCalculator
+- FNRMetricCalculator
+- FPRMetricCalculator
+
+You can implement your own `MetricCalculator`. In the example below, we define
+a metric calculator for `precision`. For that we create a `PrecisionMetricCalculator`
+class extending the `MetricCalculator` base class provided by the library and
+implementing `calculate` method.
+
+```typescript
+class PrecisionMetricCalculator extends MetricCalculator {
+  public calculate(trueValues: Float32Array, predictedValues: Float32Array): Metric {
+    const { tp, fp } = new ConfusionMatrix(trueValues, predictedValues);
+
+    return new Metric({
+      title: 'Precision',
+      value: tp / (tp + fp)
+    });
+  }
+}
+```
+
 ### Binary classification
 
 This library provides two classes to train and evaluate binary classification
@@ -83,6 +117,7 @@ Before training the model, you need to create an instance of the
   that should be fed into the model as inputs.
 - `outputFeatureExtractor` – the feature extractor to extract information that
   we want to predict.
+- `metricCalculators` – a list of metric calculators that will be used during test stage.
 
 An example can be found below:
 
@@ -100,7 +135,13 @@ const trainer = new BinaryClassificationTrainer({
     new AnnualSalaryFeatureExtractor(),
     new GenderFeatureExtractor()
   ],
-  outputFeatureExtractor: new OwnsTheCarFeatureExtractor()
+  outputFeatureExtractor: new OwnsTheCarFeatureExtractor(),
+  metricCalculators: [
+    new AccuracyMetricCalculator(),
+    new PrecisionMetricCalculator(),
+    new SpecificityMetricCalculator(),
+    new FPRMetricCalculator()
+  ]
 });
 ```
 
@@ -225,7 +266,7 @@ const ownsTheCar = await classifier.predict([0.2, 0.76, 0]);
 - [ ] Uncertainty ([#15](https://github.com/RonasIT/tfjs-node-helpers/issues/15))
 - [ ] Handle class imbalance problem ([#10](https://github.com/RonasIT/tfjs-node-helpers/issues/10))
 - [ ] Add more metrics ([#17](https://github.com/RonasIT/tfjs-node-helpers/issues/17))
-- [ ] Custom metrics ([#18](https://github.com/RonasIT/tfjs-node-helpers/issues/18))
+- [x] Custom metrics ([#18](https://github.com/RonasIT/tfjs-node-helpers/issues/18))
 - [ ] Automated tests ([#6](https://github.com/RonasIT/tfjs-node-helpers/issues/6))
 - [ ] Continuous Integration ([#11](https://github.com/RonasIT/tfjs-node-helpers/issues/11))
 - [ ] Add more examples ([#8](https://github.com/RonasIT/tfjs-node-helpers/issues/8))

packages/tfjs-node-helpers-example/src/app/app.ts

@@ -1,4 +1,15 @@
-import { BinaryClassificationTrainer, BinaryClassifier, makeChunkedDataset } from '@ronas-it/tfjs-node-helpers';
+import {
+  AccuracyMetricCalculator,
+  BinaryClassificationTrainer,
+  BinaryClassifier,
+  F1ScoreMetricCalculator,
+  FNRMetricCalculator,
+  FPRMetricCalculator,
+  makeChunkedDataset,
+  PrecisionMetricCalculator,
+  RecallMetricCalculator,
+  SpecificityMetricCalculator
+} from '@ronas-it/tfjs-node-helpers';
 import { data, layers, TensorContainer } from '@tensorflow/tfjs-node';
 import { AgeFeatureExtractor } from './feature-extractors/age';
 import { AnnualSalaryFeatureExtractor } from './feature-extractors/annual-salary';
@@ -16,10 +27,7 @@ export async function startApplication(): Promise<void> {
 
 async function train(): Promise<void> {
   const trainer = new BinaryClassificationTrainer({
-    hiddenLayers: [
-      layers.dense({ units: 128, activation: 'mish' }),
-      layers.dense({ units: 128, activation: 'mish' })
-    ],
+    hiddenLayers: [layers.dense({ units: 128, activation: 'mish' }), layers.dense({ units: 128, activation: 'mish' })],
     inputFeatureExtractors: [
       new AgeFeatureExtractor(),
       new AnnualSalaryFeatureExtractor(),
@@ -29,6 +37,15 @@ async function train(): Promise<void> {
     inputFeatureNormalizers: [
       new AgeMinMaxFeatureNormalizer(),
       new AnnualSalaryMinMaxFeatureNormalizer()
+    ],
+    metricCalculators: [
+      new AccuracyMetricCalculator(),
+      new PrecisionMetricCalculator(),
+      new F1ScoreMetricCalculator(),
+      new SpecificityMetricCalculator(),
+      new RecallMetricCalculator(),
+      new FNRMetricCalculator(),
+      new FPRMetricCalculator()
     ]
   });
 
@@ -43,23 +60,26 @@ async function train(): Promise<void> {
     trainingDataService.getTestingSamplesCount()
   ]);
 
-  const makeTrainingDataset = (): data.Dataset<TensorContainer> => makeChunkedDataset({
-    loadChunk: (skip, take) => trainingDataService.getTrainingSamples(skip, take),
-    chunkSize: 32,
-    batchSize: 32
-  });
+  const makeTrainingDataset = (): data.Dataset<TensorContainer> =>
+    makeChunkedDataset({
+      loadChunk: (skip, take) => trainingDataService.getTrainingSamples(skip, take),
+      chunkSize: 32,
+      batchSize: 32
+    });
 
-  const makeValidationDataset = (): data.Dataset<TensorContainer> => makeChunkedDataset({
-    loadChunk: (skip, take) => trainingDataService.getValidationSamples(skip, take),
-    chunkSize: 32,
-    batchSize: validationSamplesCount
-  });
+  const makeValidationDataset = (): data.Dataset<TensorContainer> =>
+    makeChunkedDataset({
+      loadChunk: (skip, take) => trainingDataService.getValidationSamples(skip, take),
+      chunkSize: 32,
+      batchSize: validationSamplesCount
+    });
 
-  const makeTestingDataset = (): data.Dataset<TensorContainer> => makeChunkedDataset({
-    loadChunk: (skip, take) => trainingDataService.getTestingSamples(skip, take),
-    chunkSize: 32,
-    batchSize: testingSamplesCount
-  });
+  const makeTestingDataset = (): data.Dataset<TensorContainer> =>
+    makeChunkedDataset({
+      loadChunk: (skip, take) => trainingDataService.getTestingSamples(skip, take),
+      chunkSize: 32,
+      batchSize: testingSamplesCount
+    });
 
   const trainingDataset = makeTrainingDataset();
   const validationDataset = makeValidationDataset();