feat(#5): add feature normalizer, refactor binary classification trainer

Author: Michael Ivonin

packages/tfjs-node-helpers-example/src/app/app.ts

@@ -6,6 +6,8 @@ import { GenderFeatureExtractor } from './feature-extractors/gender';
 import { OwnsTheCarFeatureExtractor } from './feature-extractors/owns-the-car';
 import { join } from 'node:path';
 import { TrainingDataService } from './services/training-data';
+import { AgeMinMaxFeatureNormalizer } from './feature-normalizers/age';
+import { AnnualSalaryMinMaxFeatureNormalizer } from './feature-normalizers/annual-salary';
 
 export async function startApplication(): Promise<void> {
   await train();
@@ -23,7 +25,11 @@ async function train(): Promise<void> {
       new AnnualSalaryFeatureExtractor(),
       new GenderFeatureExtractor()
     ],
-    outputFeatureExtractor: new OwnsTheCarFeatureExtractor()
+    outputFeatureExtractor: new OwnsTheCarFeatureExtractor(),
+    inputFeatureNormalizers: [
+      new AgeMinMaxFeatureNormalizer(),
+      new AnnualSalaryMinMaxFeatureNormalizer()
+    ]
   });
 
   const trainingDataService = new TrainingDataService({

packages/tfjs-node-helpers-example/src/app/feature-extractors/age.ts

@@ -6,13 +6,10 @@ export class AgeFeatureExtractor extends FeatureExtractor<DatasetItem, FeatureTy
   public featureType = FeatureType.AGE;
 
   public extract(item: DatasetItem): Feature<FeatureType> {
-    const minAge = 18;
-    const maxAge = 63;
-
     return new Feature({
       type: this.featureType,
       label: `${item.age} years`,
-      value: (item.age - minAge) / (maxAge - minAge)
+      value: item.age
     });
   }
 }

packages/tfjs-node-helpers-example/src/app/feature-extractors/annual-salary.ts

@@ -6,13 +6,10 @@ export class AnnualSalaryFeatureExtractor extends FeatureExtractor<DatasetItem,
   public featureType = FeatureType.ANNUAL_SALARY;
 
   public extract(item: DatasetItem): Feature<FeatureType> {
-    const minAnnualSalary = 15000;
-    const maxAnnualSalary = 152500;
-
     return new Feature({
       type: this.featureType,
       label: item.annual_salary.toString(),
-      value: (item.annual_salary - minAnnualSalary) / (maxAnnualSalary - minAnnualSalary)
+      value: item.annual_salary
     });
   }
 }

packages/tfjs-node-helpers-example/src/app/feature-normalizers/age.ts

@@ -0,0 +1,10 @@
+import { MinMaxFeatureNormalizer } from '@ronas-it/tfjs-node-helpers';
+import { FeatureType } from '../enums/feature-type';
+
+export class AgeMinMaxFeatureNormalizer extends MinMaxFeatureNormalizer<FeatureType> {
+  public featureType = FeatureType.AGE;
+
+  constructor() {
+    super({ min: 18, max: 63 });
+  }
+}

packages/tfjs-node-helpers-example/src/app/feature-normalizers/annual-salary.ts

@@ -0,0 +1,10 @@
+import { MinMaxFeatureNormalizer } from '@ronas-it/tfjs-node-helpers';
+import { FeatureType } from '../enums/feature-type';
+
+export class AnnualSalaryMinMaxFeatureNormalizer extends MinMaxFeatureNormalizer<FeatureType> {
+  public featureType = FeatureType.ANNUAL_SALARY;
+
+  constructor() {
+    super({ min: 15000, max: 152500 });
+  }
+}

packages/tfjs-node-helpers-example/src/app/services/training-data.ts

@@ -1,9 +1,16 @@
-import { extractFeatures, Sample, splitSamplesIntoTrainingValidationTestForBinaryClassification } from '@ronas-it/tfjs-node-helpers';
+import {
+  extractFeatures,
+  normalizeFeatures,
+  Sample,
+  splitSamplesIntoTrainingValidationTestForBinaryClassification
+} from '@ronas-it/tfjs-node-helpers';
 import { AgeFeatureExtractor } from '../feature-extractors/age';
 import { AnnualSalaryFeatureExtractor } from '../feature-extractors/annual-salary';
 import { GenderFeatureExtractor } from '../feature-extractors/gender';
 import { OwnsTheCarFeatureExtractor } from '../feature-extractors/owns-the-car';
 import dataset from '../../assets/data.json';
+import { AgeMinMaxFeatureNormalizer } from '../feature-normalizers/age';
+import { AnnualSalaryMinMaxFeatureNormalizer } from '../feature-normalizers/annual-salary';
 
 export class TrainingDataService {
   private simulatedDelayMs: number;
@@ -16,7 +23,7 @@ export class TrainingDataService {
   }
 
   public async initialize(): Promise<void> {
-    const samples = await extractFeatures({
+    const extracts = await extractFeatures({
       data: dataset,
       inputFeatureExtractors: [
         new AgeFeatureExtractor(),
@@ -26,6 +33,14 @@ export class TrainingDataService {
       outputFeatureExtractor: new OwnsTheCarFeatureExtractor()
     });
 
+    const samples = await normalizeFeatures({
+      extracts,
+      inputFeatureNormalizers: [
+        new AgeMinMaxFeatureNormalizer(),
+        new AnnualSalaryMinMaxFeatureNormalizer()
+      ]
+    });
+
     const { trainingSamples, validationSamples, testingSamples } = splitSamplesIntoTrainingValidationTestForBinaryClassification(samples);
 
     this.trainingSamples = trainingSamples;

packages/tfjs-node-helpers/src/classification/binary-classification-trainer.ts

@@ -19,13 +19,15 @@ import { prepareDatasetsForBinaryClassification } from '../feature-engineering/p
 import { ConfusionMatrix } from '../testing/confusion-matrix';
 import { Metrics } from '../testing/metrics';
 import { binarize } from '../utils/binarize';
+import { FeatureNormalizer } from '../feature-engineering/feature-normalizer';
 
 export type BinaryClassificationTrainerOptions = {
   batchSize?: number;
   epochs?: number;
   patience?: number;
   inputFeatureExtractors?: Array<FeatureExtractor<any, any>>;
   outputFeatureExtractor?: FeatureExtractor<any, any>;
+  inputFeatureNormalizers?: Array<FeatureNormalizer<any>>;
   model?: LayersModel;
   hiddenLayers?: Array<layers.Layer>;
   optimizer?: string | Optimizer;
@@ -39,6 +41,7 @@ export class BinaryClassificationTrainer {
   protected tensorBoardLogsDirectory?: string;
   protected inputFeatureExtractors?: Array<FeatureExtractor<any, any>>;
   protected outputFeatureExtractor?: FeatureExtractor<any, any>;
+  protected inputFeatureNormalizers?: Array<FeatureNormalizer<any>>;
   protected model!: LayersModel;
 
   protected static DEFAULT_BATCH_SIZE: number = 32;
@@ -52,6 +55,7 @@ export class BinaryClassificationTrainer {
     this.tensorBoardLogsDirectory = options.tensorBoardLogsDirectory;
     this.inputFeatureExtractors = options.inputFeatureExtractors;
     this.outputFeatureExtractor = options.outputFeatureExtractor;
+    this.inputFeatureNormalizers = options.inputFeatureNormalizers;
 
     this.initializeModel(options);
   }
@@ -63,7 +67,7 @@ export class BinaryClassificationTrainer {
     testingDataset,
     printTestingResults
   }: {
-    data?: Array<any>,
+    data?: Array<any>;
     trainingDataset?: data.Dataset<TensorContainer>;
     validationDataset?: data.Dataset<TensorContainer>;
     testingDataset?: data.Dataset<TensorContainer>;
@@ -90,14 +94,19 @@ export class BinaryClassificationTrainer {
       validationDataset === undefined ||
       testingDataset === undefined
     ) {
-      if (this.inputFeatureExtractors === undefined || this.outputFeatureExtractor === undefined) {
-        throw new Error('trainingDataset, validationDataset and testingDataset are required when inputFeatureExtractors and outputFeatureExtractor are not provided!');
+      if (
+        this.inputFeatureExtractors === undefined ||
+        this.outputFeatureExtractor === undefined ||
+        this.inputFeatureNormalizers === undefined
+      ) {
+        throw new Error('trainingDataset, validationDataset and testingDataset are required when inputFeatureExtractors, outputFeatureExtractor and inputFeatureNormalizers are not provided!');
       }
 
       const datasets = await prepareDatasetsForBinaryClassification({
         data: data as Array<any>,
         inputFeatureExtractors: this.inputFeatureExtractors,
         outputFeatureExtractor: this.outputFeatureExtractor,
+        inputFeatureNormalizers: this.inputFeatureNormalizers,
         batchSize: this.batchSize
       });
 

packages/tfjs-node-helpers/src/feature-engineering/extract-features.ts

@@ -1,5 +1,10 @@
-import { Sample } from '../training/sample';
 import { FeatureExtractor } from './feature-extractor';
+import { Feature } from './feature';
+
+export type DataItemExtract<T> = {
+  inputFeatures: Array<Feature<T>>;
+  outputFeature: Feature<T>;
+};
 
 export const extractFeatures = async <D, T>({
   data,
@@ -9,8 +14,8 @@ export const extractFeatures = async <D, T>({
   data: Array<D>;
   inputFeatureExtractors: Array<FeatureExtractor<D, T>>;
   outputFeatureExtractor: FeatureExtractor<D, T>;
-}): Promise<Array<Sample>> => {
-  const samples = [];
+}): Promise<Array<DataItemExtract<T>>> => {
+  const extracts = [];
 
   for (const dataItem of data) {
     const [inputFeatures, outputFeature] = await Promise.all([
@@ -22,11 +27,8 @@ export const extractFeatures = async <D, T>({
       outputFeatureExtractor.extract(dataItem)
     ]);
 
-    const input = inputFeatures.map((feature) => feature.value);
-    const output = [outputFeature.value];
-
-    samples.push({ input, output });
+    extracts.push({ inputFeatures, outputFeature });
   }
 
-  return samples;
+  return extracts;
 }

packages/tfjs-node-helpers/src/feature-engineering/feature-normalizer.ts

@@ -0,0 +1,7 @@
+import { Feature } from './feature';
+
+export abstract class FeatureNormalizer<T> {
+  public abstract featureType: T;
+
+  public abstract normalize(feature: Feature<T>): Feature<T> | Promise<Feature<T>>;
+}

packages/tfjs-node-helpers/src/feature-engineering/min-max-feature-normalizer.ts

@@ -0,0 +1,21 @@
+import { FeatureNormalizer } from './feature-normalizer';
+import { Feature } from './feature';
+
+export abstract class MinMaxFeatureNormalizer<T> extends FeatureNormalizer<T> {
+  private min: number;
+  private max: number;
+
+  constructor({ min, max }: { min: number; max: number }) {
+    super();
+
+    this.min = min;
+    this.max = max;
+  }
+
+  public normalize(feature: Feature<T>): Feature<T> | Promise<Feature<T>> {
+    return new Feature({
+      ...feature,
+      value: (feature.value - this.min) / (this.max - this.min)
+    });
+  }
+}