finally... migration | closes #171

Author: GalvinPython

package.json

@@ -26,8 +26,9 @@
     "db:push:dev": "cross-env NODE_ENV=development bun drizzle-kit push",
     "db:push:staging": "cross-env NODE_ENV=staging bun drizzle-kit push",
     "db:push:prod": "bun drizzle-kit push",
-    "db:migrate:staging": "bun src/utils/migratedb.ts --staging",
-    "db:migrate:prod": "bun src/utils/migratedb.ts",
+    "db:migrate:staging": "bun src/db/migratedb.ts --staging",
+    "db:migrate:prod": "bun src/db/migratedb.ts",
+    "db:staging:reset": "bun src/db/resetStagingDatabase.ts --staging",
     "dev": "concurrently --names \"WEB,API,BOT\" --prefix-colors \"blue,green,magenta\" \"cd web && bun dev\" \"cd api && cargo watch -x \\\"run -- --dev\\\"\" \"bun --watch . --dev\"",
     "dev:bot": "bun --watch . --dev",
     "test:jetstream": "bun --watch src/utils/bluesky/jetstream.ts",

src/db/migratedb.ts

@@ -0,0 +1,129 @@
+import path from "path";
+
+import { Database } from "bun:sqlite";
+import { drizzle as drizzleSqlite } from "drizzle-orm/bun-sqlite";
+import { drizzle as drizzlePostgres } from "drizzle-orm/node-postgres";
+import { Client } from "pg";
+
+import { config } from "../config";
+import { getStreamerName } from "../utils/twitch/getStreamerName";
+import checkIfChannelIdIsValid from "../utils/youtube/checkIfChannelIdIsValid";
+import { Platform } from "../types/types.d";
+import { getTwitchToken } from "../utils/twitch/auth";
+
+import * as sqliteSchema from "./schemaSqlite";
+import * as pgSchema from "./schema";
+import { addNewChannelToTrack } from "./youtube";
+import { addNewStreamerToTrack } from "./twitch";
+import { discordAddGuildTrackingUser } from "./discord";
+
+// Get Twitch token
+if (!(await getTwitchToken())) {
+    throw new Error("Error getting Twitch token");
+}
+
+// SQLite connection
+const sqlite = new Database(path.resolve(process.cwd(), "db-prod.sqlite3"));
+const sqliteDb = drizzleSqlite(sqlite, { schema: sqliteSchema });
+
+// Postgres connection
+const client = new Client({ connectionString: config.databaseUrl });
+
+await client.connect();
+const pgDb = drizzlePostgres(client, { schema: pgSchema });
+
+console.log("📋 Starting migration...", config.databaseUrl);
+
+async function migrate() {
+    // 1. Bot info
+    const botInfo = await sqliteDb.select().from(sqliteSchema.sqliteBotInfo);
+
+    for (const row of botInfo) {
+        console.log(
+            `📋 Migrating bot info: guildsTotal=${row.totalServers ?? 0}, totalMembers=${row.totalMembers ?? 0}`,
+        );
+        await pgDb.insert(pgSchema.dbBotInfoTable).values({
+            guildsTotal: row.totalServers ?? 0,
+            totalMembers: row.totalMembers ?? 0,
+        });
+    }
+
+    // 2. Discord guilds
+    const discordRows = await sqliteDb
+        .select()
+        .from(sqliteSchema.sqliteDiscord);
+
+    for (const row of discordRows) {
+        console.log("📋 Migrating discord guild:", row.guildId);
+        await pgDb
+            .insert(pgSchema.dbDiscordTable)
+            .values({ guildId: row.guildId ?? "" })
+            .onConflictDoNothing();
+    }
+
+    // 3. Twitch
+    const twitchRows = await sqliteDb.select().from(sqliteSchema.sqliteTwitch);
+
+    for (const row of twitchRows) {
+        const twitchChannelName = await getStreamerName(row.twitchChannelId);
+
+        if (!twitchChannelName) {
+            console.log(
+                `⚠️  Skipping Twitch channel ID ${row.twitchChannelId} as it no longer exists.`,
+            );
+            continue;
+        }
+        await addNewStreamerToTrack(
+            row.twitchChannelId,
+            Boolean(row.isLive),
+            twitchChannelName,
+        );
+    }
+
+    // 4. YouTube
+    const youtubeRows = await sqliteDb
+        .select()
+        .from(sqliteSchema.sqliteYouTube);
+
+    for (const row of youtubeRows) {
+        if (!checkIfChannelIdIsValid(row.youtubeChannelId)) {
+            console.log(
+                `⚠️  Skipping YouTube channel ID ${row.youtubeChannelId} as it is not a valid channel ID.`,
+            );
+            continue;
+        }
+        await addNewChannelToTrack(row.youtubeChannelId);
+    }
+
+    // 5. Guild Subscriptions (after Twitch/YouTube exist!)
+    for (const row of discordRows) {
+        if (row.guildPlatform === "twitch") {
+            await discordAddGuildTrackingUser(
+                row.guildId,
+                Platform.Twitch,
+                row.platformUserId,
+                row.guildChannelId,
+                row.guildPingRole,
+                false,
+            );
+        } else if (row.guildPlatform === "youtube") {
+            await discordAddGuildTrackingUser(
+                row.guildId,
+                Platform.YouTube,
+                row.platformUserId,
+                row.guildChannelId,
+                row.guildPingRole,
+                false,
+                true,
+                true,
+                true,
+            );
+        }
+    }
+}
+
+await migrate();
+
+// Sometimes the connection pool doesn't close properly
+// so we need to force it to close by... doing this
+process.exit(0);

src/db/resetStagingDatabase.ts

@@ -0,0 +1,37 @@
+import { Client } from "pg";
+
+import { config } from "../config";
+
+// Connect to Postgres
+const client = new Client({ connectionString: config.databaseUrl });
+
+await client.connect();
+
+console.log(`Truncating all tables in database: ${config.databaseUrl}`);
+
+try {
+    // 1. Get all table names in the public schema
+    const { rows: tables } = await client.query(`
+    SELECT table_name
+    FROM information_schema.tables
+    WHERE table_schema = 'public' AND table_type = 'BASE TABLE';
+  `);
+
+    if (tables.length === 0) {
+        console.log("No tables found to truncate");
+    } else {
+        // 2. Build comma-separated list of tables
+        const tableNames = tables.map((t) => `"${t.table_name}"`).join(", ");
+
+        // 3. Truncate all tables
+        await client.query(
+            `TRUNCATE TABLE ${tableNames} RESTART IDENTITY CASCADE;`,
+        );
+        console.log(`Truncated ${tables.length} tables: ${tableNames}`);
+    }
+} catch (err) {
+    console.error("❌ Error truncating tables:", err);
+} finally {
+    await client.end();
+    process.exit(0);
+}

src/db/schemaSqlite.ts

@@ -0,0 +1,24 @@
+import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core";
+
+export const sqliteBotInfo = sqliteTable("bot_info", {
+    totalServers: integer("total_servers").notNull(),
+    totalMembers: integer("total_members").notNull(),
+});
+
+export const sqliteDiscord = sqliteTable("discord", {
+    guildId: text("guild_id").notNull(),
+    guildChannelId: text("guild_channel_id").notNull(),
+    guildPlatform: text("guild_platform").notNull(),
+    platformUserId: text("platform_user_id").notNull(),
+    guildPingRole: text("guild_ping_role"),
+});
+
+export const sqliteTwitch = sqliteTable("twitch", {
+    twitchChannelId: text("twitch_channel_id").notNull(),
+    isLive: integer("is_live").notNull(),
+});
+
+export const sqliteYouTube = sqliteTable("youtube", {
+    youtubeChannelId: text("youtube_channel_id").notNull(),
+    latestVideoId: text("latest_video_id"),
+});