initial fillStorage

Author: phartenfeller

src/lib/server/db/index.ts

@@ -22,6 +22,12 @@ type QueryWrapper = {
 
 export type QueryResult<T> = { data: T } & QueryWrapper;
 
+export function getCustomerCount(): number {
+	const stmt = db.prepare(`SELECT COUNT(*) as "cnt" FROM customers`);
+	const data = stmt.get() as { cnt: number };
+	return data.cnt;
+}
+
 export function getCustomers({ offset = 0, limit = 50 }): QueryResult<Customer[]> {
 	const stmt = db.prepare(`
   SELECT CustomerID as "id"
@@ -40,6 +46,6 @@ export function getCustomers({ offset = 0, limit = 50 }): QueryResult<Customer[]
 
 	return {
 		data,
-		moreRows: data.length === limit
+		moreRows: getCustomerCount() > offset + limit
 	};
 }

src/lib/sqlite/initStorages.ts

@@ -4,7 +4,10 @@ import {
 	WorkerMessageTypes,
 	type WorkerMessage,
 	type TableExistsResponseData,
-	type CreateTableRequestData
+	type CreateTableRequestData,
+	type DataRow,
+	type FillStorageRequestData,
+	type FillStorageResponseData
 } from './types';
 
 const storages = ['customers_v1'];
@@ -15,6 +18,12 @@ async function getStructure(storage: string): Promise<TableStructure> {
 	return data;
 }
 
+async function getData(storage: string, offset: number, limit: number) {
+	const res = await fetch(`/api/data/${storage}/data?offset=${offset}&limit=${limit}`);
+	const data = (await res.json()) as { data: DataRow[]; moreRows: boolean };
+	return data;
+}
+
 async function createStorage(storage: string, structure: TableStructure) {
 	const res = (await sendMsgToWorker({
 		storageId: storage,
@@ -26,6 +35,32 @@ async function createStorage(storage: string, structure: TableStructure) {
 	if (res.data.errorMsg) throw new Error(res.data.errorMsg);
 }
 
+async function fillStorage(storage: string, structure: TableStructure) {
+	const PAGE_SIZE = 100;
+	let currOffset = 0;
+	let fetchMore = false;
+
+	do {
+		const { data, moreRows } = await getData(storage, currOffset, PAGE_SIZE);
+		console.log(`Fetched ${data.length} rows from ${storage} at offset ${currOffset}`, data);
+
+		const res = (await sendMsgToWorker({
+			storageId: storage,
+			type: WorkerMessageTypes.FILL_STORAGE,
+			expectedType: WorkerMessageTypes.FILL_STORAGE_RESPONSE,
+			data: { rows: data, structure: structure } as FillStorageRequestData
+		})) as WorkerMessage<FillStorageResponseData>;
+
+		if (res.data.errorMsg) {
+			console.error(`Error filling storage ${storage} at offset ${currOffset}`, res.data.errorMsg);
+			throw new Error(res.data.errorMsg);
+		}
+
+		currOffset += PAGE_SIZE;
+		fetchMore = moreRows;
+	} while (fetchMore);
+}
+
 export default async function initStorages() {
 	for (const storageId of storages) {
 		const res = (await sendMsgToWorker({
@@ -39,10 +74,16 @@ export default async function initStorages() {
 
 		if (res.data.tableExists) {
 			console.log(`Table ${storageId} exists. Has data: ${res.data.hasData}`);
+
+			if (!res.data.hasData) {
+				const structure = await getStructure(storageId);
+				await fillStorage(storageId, structure);
+			}
 		} else {
 			const structure = await getStructure(storageId);
 			console.log(`Table ${storageId} does not exist. Creating...`, structure);
 			await createStorage(storageId, structure);
+			await fillStorage(storageId, structure);
 		}
 	}
 }

src/lib/sqlite/types.ts

@@ -6,9 +6,13 @@ export enum WorkerMessageTypes {
 	TABLE_EXISTS = 'TABLE_EXISTS',
 	TABLE_EXISTS_RESPONSE = 'TABLE_EXISTS_RESPONSE',
 	CREATE_TABLE = 'CREATE_TABLE',
-	CREATE_TABLE_RESPONSE = 'CREATE_TABLE_RESPONSE'
+	CREATE_TABLE_RESPONSE = 'CREATE_TABLE_RESPONSE',
+	FILL_STORAGE = 'FILL_STORAGE',
+	FILL_STORAGE_RESPONSE = 'FILL_STORAGE_RESPONSE'
 }
 
+export type DataRow = { [key: string]: string | number | boolean | null };
+
 export type WorkerMessageBase = {
 	messageId: string;
 	type: WorkerMessageTypes;
@@ -30,3 +34,12 @@ export type CreateTableRequestData = {
 export type CreateTableResponseData = {
 	errorMsg?: string;
 };
+
+export type FillStorageRequestData = {
+	structure: TableStructure;
+	rows: DataRow[];
+};
+
+export type FillStorageResponseData = {
+	errorMsg?: string;
+};

src/lib/sqlite/worker/storageHandlers.ts

@@ -1,11 +1,15 @@
 import type {
 	CreateTableRequestData,
 	CreateTableResponseData,
+	FillStorageRequestData,
+	FillStorageResponseData,
 	TableExistsResponseData,
 	WorkerMessage
 } from '../types';
 import { db } from './initDb';
+import genInsertSql from './util/genInsertSql';
 import genTabSrc from './util/genTabSrc';
+import getBindObject from './util/getBindObject';
 import { tableExists, tableHasData } from './util/tableInfo';
 
 export function handleTableExists(data: WorkerMessage<unknown>): TableExistsResponseData {
@@ -53,3 +57,28 @@ export function handleCreateTable(
 		};
 	}
 }
+
+export function handleFillStorage(
+	msg: WorkerMessage<FillStorageRequestData>
+): FillStorageResponseData {
+	try {
+		const { storageId, data } = msg;
+		const { rows, structure } = data;
+
+		const src = genInsertSql(storageId, structure);
+		console.log('Insert sql:', src);
+
+		for (const row of rows) {
+			db.exec({ sql: src, bind: getBindObject(row) });
+		}
+
+		return {};
+	} catch (err) {
+		const msg = `Error filling storage: ${err}`;
+		console.error(msg);
+
+		return {
+			errorMsg: msg
+		};
+	}
+}

src/lib/sqlite/worker/util/genInsertSql.ts

@@ -0,0 +1,17 @@
+import type { TableStructure } from '../../../../routes/api/data/types';
+
+export default function genInsertSql(storageId: string, structure: TableStructure) {
+	let statement = `Insert Into ${storageId} (`;
+
+	const atomics: string[] = [];
+
+	for (const col of structure.columns) {
+		atomics.push(col.name);
+	}
+
+	statement += `   ${atomics.join(', ')}    ) Values ( ${atomics
+		.map((a) => `$${a}`)
+		.join(', ')} );`;
+
+	return statement;
+}

src/lib/sqlite/worker/util/getBindObject.ts

@@ -0,0 +1,11 @@
+import type { DataRow } from '$lib/sqlite/types';
+
+export default function getBindObject(data: DataRow): DataRow {
+	const bindObject: DataRow = {};
+
+	for (const key in data) {
+		bindObject[`$${key}`] = data[key];
+	}
+
+	return bindObject;
+}

src/lib/sqlite/worker/worker.ts

@@ -4,10 +4,12 @@ import {
 	type WorkerMessage,
 	type TableExistsResponseData,
 	type CreateTableResponseData,
-	type CreateTableRequestData
+	type CreateTableRequestData,
+	type FillStorageRequestData,
+	type FillStorageResponseData
 } from '../types';
 import { initDb } from './initDb';
-import { handleCreateTable, handleTableExists } from './storageHandlers';
+import { handleCreateTable, handleFillStorage, handleTableExists } from './storageHandlers';
 
 console.log('worker loaded');
 
@@ -61,6 +63,22 @@ function sendMsgToMain(obj: WorkerMessage<unknown>) {
 				};
 				sendMsgToMain(createTableResult);
 				break;
+
+			case WorkerMessageTypes.FILL_STORAGE:
+				const fillStorageData = await handleFillStorage(
+					data as WorkerMessage<FillStorageRequestData>
+				);
+
+				const fillStorageResult: WorkerMessage<FillStorageResponseData> = {
+					type: WorkerMessageTypes.FILL_STORAGE_RESPONSE,
+					messageId: data.messageId,
+					storageId: data.storageId,
+					data: fillStorageData
+				};
+
+				sendMsgToMain(fillStorageResult);
+				break;
+
 			default:
 				throw new Error(`Unknown message type: ${data.type}`);
 		}