Merge pull request #21 from RonasIT/14-load-data-asynchronously-in-the-example-application

Load data asynchronously in the example application

Author: astorozhevsky

packages/tfjs-node-helpers-example/src/app/app.ts

@@ -1,11 +1,11 @@
-import { BinaryClassificationTrainer, BinaryClassifier } from '@ronas-it/tfjs-node-helpers';
-import { layers } from '@tensorflow/tfjs-node';
+import { BinaryClassificationTrainer, BinaryClassifier, makeChunkedDataset } from '@ronas-it/tfjs-node-helpers';
+import { data, layers, TensorContainer } from '@tensorflow/tfjs-node';
 import { AgeFeatureExtractor } from './feature-extractors/age';
 import { AnnualSalaryFeatureExtractor } from './feature-extractors/annual-salary';
 import { GenderFeatureExtractor } from './feature-extractors/gender';
 import { OwnsTheCarFeatureExtractor } from './feature-extractors/owns-the-car';
 import { join } from 'node:path';
-import data from '../assets/data.json';
+import { TrainingDataService } from './services/training-data';
 
 export async function startApplication(): Promise<void> {
   await train();
@@ -26,8 +26,43 @@ async function train(): Promise<void> {
     outputFeatureExtractor: new OwnsTheCarFeatureExtractor()
   });
 
+  const trainingDataService = new TrainingDataService({
+    simulatedDelayMs: 100
+  });
+
+  await trainingDataService.initialize();
+
+  const [validationSamplesCount, testingSamplesCount] = await Promise.all([
+    trainingDataService.getValidationSamplesCount(),
+    trainingDataService.getTestingSamplesCount()
+  ]);
+
+  const makeTrainingDataset = (): data.Dataset<TensorContainer> => makeChunkedDataset({
+    loadChunk: (skip, take) => trainingDataService.getTrainingSamples(skip, take),
+    chunkSize: 32,
+    batchSize: 32
+  });
+
+  const makeValidationDataset = (): data.Dataset<TensorContainer> => makeChunkedDataset({
+    loadChunk: (skip, take) => trainingDataService.getValidationSamples(skip, take),
+    chunkSize: 32,
+    batchSize: validationSamplesCount
+  });
+
+  const makeTestingDataset = (): data.Dataset<TensorContainer> => makeChunkedDataset({
+    loadChunk: (skip, take) => trainingDataService.getTestingSamples(skip, take),
+    chunkSize: 32,
+    batchSize: testingSamplesCount
+  });
+
+  const trainingDataset = makeTrainingDataset();
+  const validationDataset = makeValidationDataset();
+  const testingDataset = makeTestingDataset();
+
   await trainer.trainAndTest({
-    data,
+    trainingDataset,
+    validationDataset,
+    testingDataset,
     printTestingResults: true
   });
 

packages/tfjs-node-helpers-example/src/app/services/training-data.ts

@@ -0,0 +1,71 @@
+import { extractFeatures, Sample, splitSamplesIntoTrainingValidationTestForBinaryClassification } from '@ronas-it/tfjs-node-helpers';
+import { AgeFeatureExtractor } from '../feature-extractors/age';
+import { AnnualSalaryFeatureExtractor } from '../feature-extractors/annual-salary';
+import { GenderFeatureExtractor } from '../feature-extractors/gender';
+import { OwnsTheCarFeatureExtractor } from '../feature-extractors/owns-the-car';
+import dataset from '../../assets/data.json';
+
+export class TrainingDataService {
+  private simulatedDelayMs: number;
+  private trainingSamples: Array<Sample>;
+  private validationSamples: Array<Sample>;
+  private testingSamples: Array<Sample>;
+
+  constructor({ simulatedDelayMs }: { simulatedDelayMs: number }) {
+    this.simulatedDelayMs = simulatedDelayMs;
+  }
+
+  public async initialize(): Promise<void> {
+    const samples = await extractFeatures({
+      data: dataset,
+      inputFeatureExtractors: [
+        new AgeFeatureExtractor(),
+        new AnnualSalaryFeatureExtractor(),
+        new GenderFeatureExtractor()
+      ],
+      outputFeatureExtractor: new OwnsTheCarFeatureExtractor()
+    });
+
+    const { trainingSamples, validationSamples, testingSamples } = splitSamplesIntoTrainingValidationTestForBinaryClassification(samples);
+
+    this.trainingSamples = trainingSamples;
+    this.validationSamples = validationSamples;
+    this.testingSamples = testingSamples;
+  }
+
+  public async getTrainingSamples(skip: number, take: number): Promise<Array<Sample>> {
+    await this.simulateDelay();
+
+    return this.trainingSamples.slice(skip, skip + take);
+  }
+
+  public async getValidationSamples(skip: number, take: number): Promise<Array<Sample>> {
+    await this.simulateDelay();
+
+    return this.validationSamples.slice(skip, skip + take);
+  }
+
+  public async getTestingSamples(skip: number, take: number): Promise<Array<Sample>> {
+    await this.simulateDelay();
+
+    return this.testingSamples.slice(skip, skip + take);
+  }
+
+  public async getValidationSamplesCount(): Promise<number> {
+    await this.simulateDelay();
+
+    return this.validationSamples.length;
+  }
+
+  public async getTestingSamplesCount(): Promise<number> {
+    await this.simulateDelay();
+
+    return this.testingSamples.length;
+  }
+
+  private async simulateDelay(): Promise<void> {
+    if (this.simulatedDelayMs > 0) {
+      await new Promise((resolve) => setTimeout(resolve, this.simulatedDelayMs));
+    }
+  }
+}