Skip to content

Commit 7c68c4c

Browse files
GalvinPythonGalvinPython
committed
migrate db
1 parent ba95662 commit 7c68c4c

File tree

9 files changed

+213
-49
lines changed

9 files changed

+213
-49
lines changed

‎.gitignore‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@ node_modules/
55
/backups
66
*.db
77
*.sqlite
8-
*.sqlite3
9-
*.sql
8+
*.sqlite3*
9+
*.sql
10+
dbtests.ts

‎src/events/ready.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ client.once(Events.ClientReady, async (bot) => {
3737
fetchLatestUploads();
3838
setInterval(updatePresence, 60000);
3939
setInterval(fetchLatestUploads, config.updateIntervalYouTube as number);
40-
checkIfStreamersAreLive();
41-
setInterval(checkIfStreamersAreLive, config.updateIntervalTwitch as number);
40+
// One at a time
41+
// checkIfStreamersAreLive();
42+
// setInterval(checkIfStreamersAreLive, config.updateIntervalTwitch as number);
4243
});

‎src/types/database.d.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
export interface dbYouTube {
2-
channelId: string;
3-
lastVideoId: string;
2+
youtube_channel_id: string;
3+
latest_video_id: string | null;
4+
latest_video_id_updated: Date | null;
5+
latest_short_id: string | null;
6+
latest_short_id_updated: Date | null;
7+
latest_stream_id: string | null;
8+
latest_stream_id_updated: Date | null;
9+
youtube_channel_is_live: boolean;
410
}
511

612
export interface dbTwitch {

‎src/types/innertube.d.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
export type InnertubeSearchRequest = {
1+
export interface InnertubeSearchRequest {
22
contents: {
33
twoColumnSearchResultsRenderer: {
44
primaryContents: {
@@ -163,4 +163,4 @@ export type InnertubeSearchRequest = {
163163
};
164164
};
165165
};
166-
};
166+
}

‎src/utils/database.ts‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -113,20 +113,6 @@ export async function addNewGuildToTrackChannel(
113113
}
114114
}
115115

116-
export async function getAllChannelsToTrack() {
117-
const query = `SELECT * FROM youtube`;
118-
119-
try {
120-
const statement = db.prepare(query);
121-
const results = statement.all() as dbYouTube[];
122-
123-
return results;
124-
} catch (err) {
125-
console.error("Error getting all channels to track:", err);
126-
throw err;
127-
}
128-
}
129-
130116
export async function getGuildsTrackingChannel(channelId: string) {
131117
const query = `SELECT * FROM discord WHERE platform_user_id = ?`;
132118

‎src/utils/db/init.ts‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,9 @@ export default async function initTables(): Promise<boolean> {
9494

9595
const createBotInfoNotificationsTimingsTable = `
9696
CREATE TABLE IF NOT EXISTS bot_info_notifications_timings (
97-
time DATETIME NOT NULL,
97+
time TIMESTAMP NOT NULL,
9898
channel_id TEXT NOT NULL REFERENCES youtube(youtube_channel_id),
99-
time_ms INTEGER NOT NULL DEFAULT 0,
99+
time_ms INTEGER NOT NULL DEFAULT 0
100100
);
101101
`;
102102

@@ -126,43 +126,45 @@ export default async function initTables(): Promise<boolean> {
126126
`;
127127

128128
try {
129-
await pool.query(createDiscordTable);
129+
const client = await pool.connect();
130+
131+
await client.query(createDiscordTable);
130132
console.log("Discord table created");
131133

132-
await pool.query(createBlueskyTable);
134+
await client.query(createBlueskyTable);
133135
console.log("Bluesky table created");
134136

135-
await pool.query(createYouTubeTable);
137+
await client.query(createYouTubeTable);
136138
console.log("YouTube table created");
137139

138-
await pool.query(createTwitchTable);
140+
await client.query(createTwitchTable);
139141
console.log("Twitch table created");
140142

141-
await pool.query(createGuildBlueskySubscriptionsTable);
143+
await client.query(createGuildBlueskySubscriptionsTable);
142144
console.log("Guild Bluesky Subscriptions table created");
143145

144-
await pool.query(createGuildYouTubeSubscriptionsTable);
146+
await client.query(createGuildYouTubeSubscriptionsTable);
145147
console.log("Guild YouTube Subscriptions table created");
146148

147-
await pool.query(createGuildTwitchSubscriptionsTable);
149+
await client.query(createGuildTwitchSubscriptionsTable);
148150
console.log("Guild Twitch Subscriptions table created");
149151

150-
await pool.query(createBotInfoTable);
152+
await client.query(createBotInfoTable);
151153
console.log("Bot Info table created");
152154

153-
await pool.query(createBotInfoNotificationsTable);
155+
await client.query(createBotInfoNotificationsTable);
154156
console.log("Bot Info Notifications table created");
155157

156-
await pool.query(createBotInfoNotificationsTimingsTable);
158+
await client.query(createBotInfoNotificationsTimingsTable);
157159
console.log("Bot Info Notifications Timings table created");
158160

159-
await pool.query(createBotInfoTopChannelsTable);
161+
await client.query(createBotInfoTopChannelsTable);
160162
console.log("Bot Info Top Channels table created");
161163

162-
await pool.query(createBotInfoTopGuildsTable);
164+
await client.query(createBotInfoTopGuildsTable);
163165
console.log("Bot Info Top Guilds table created");
164166

165-
await pool.query(createAuditLogsTable);
167+
await client.query(createAuditLogsTable);
166168
console.log("Audit Logs table created");
167169

168170
return true;

‎src/utils/db/youtube.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import type { dbYouTube } from "../../types/database";
2+
3+
import { pool } from "../database";
4+
5+
export async function dbYouTubeGetAllChannelsToTrack(): Promise<
6+
dbYouTube[] | []
7+
> {
8+
const query = `SELECT * FROM youtube`;
9+
10+
try {
11+
const client = await pool.connect();
12+
const result = await client.query(query);
13+
14+
client.release();
15+
16+
return result.rows as dbYouTube[];
17+
} catch (err) {
18+
console.error("Error getting all channels to track:", err);
19+
20+
return [];
21+
}
22+
}

‎src/utils/migratedb.ts‎

Lines changed: 135 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,138 @@ import path from "path";
22

33
import { Database } from "bun:sqlite";
44

5-
const db = new Database(path.resolve(process.cwd(), "db.sqlite3"));
5+
import { pool } from "./database";
6+
7+
const sqliteDb = new Database(path.resolve(process.cwd(), "db.sqlite3"));
8+
// const sqliteDb = new Database(path.resolve(process.cwd(), "db-prod.sqlite3"));
9+
10+
const client = await pool.connect();
11+
12+
console.log("▶️ Starting migration...");
13+
14+
// Load SQLite data
15+
const discordData = sqliteDb.query("SELECT * FROM discord").all();
16+
17+
console.log(
18+
`🔃 Loaded ${discordData.length} rows from discord table in SQLite`,
19+
);
20+
const youtubeData = sqliteDb.query("SELECT * FROM youtube").all();
21+
22+
console.log(
23+
`🔃 Loaded ${youtubeData.length} rows from youtube table in SQLite`,
24+
);
25+
const twitchData = sqliteDb.query("SELECT * FROM twitch").all();
26+
27+
console.log(`🔃 Loaded ${twitchData.length} rows from twitch table in SQLite`);
28+
const botInfoData = sqliteDb.query("SELECT * FROM bot_info").all();
29+
30+
console.log(
31+
`🔃 Loaded ${botInfoData.length} rows from bot_info table in SQLite`,
32+
);
33+
34+
// Insert into discord table
35+
const uniqueGuilds = new Set(discordData.map((d: any) => d.guild_id));
36+
37+
for (const guild_id of uniqueGuilds) {
38+
await client.query(
39+
`
40+
INSERT INTO discord (guild_id, is_dm, allowed_public_sharing)
41+
VALUES ($1, false, false)
42+
ON CONFLICT (guild_id) DO NOTHING
43+
`,
44+
[guild_id],
45+
);
46+
console.log(`➡️ Inserted guild ${guild_id} into discord table`);
47+
}
48+
49+
// Insert into twitch table
50+
for (const row of twitchData as any) {
51+
await client.query(
52+
`
53+
INSERT INTO twitch (twitch_channel_id, twitch_channel_is_live)
54+
VALUES ($1, $2)
55+
ON CONFLICT (twitch_channel_id) DO NOTHING
56+
`,
57+
[row.twitch_channel_id, row.is_live === 1],
58+
);
59+
console.log(
60+
`➡️ Inserted twitch channel ${row.twitch_channel_id} into twitch table`,
61+
);
62+
}
63+
64+
// Insert into youtube table
65+
for (const row of youtubeData as any) {
66+
await client.query(
67+
`
68+
INSERT INTO youtube (youtube_channel_id, latest_video_id)
69+
VALUES ($1, $2)
70+
ON CONFLICT (youtube_channel_id) DO UPDATE SET latest_video_id = EXCLUDED.latest_video_id
71+
`,
72+
[row.youtube_channel_id, row.latest_video_id],
73+
);
74+
console.log(
75+
`➡️ Inserted youtube channel ${row.youtube_channel_id} into youtube table`,
76+
);
77+
}
78+
79+
// Insert into guild_twitch_subscriptions and guild_youtube_subscriptions
80+
for (const row of discordData as any) {
81+
const is_dm = false;
82+
83+
if (row.guild_platform === "twitch") {
84+
await client.query(
85+
`
86+
INSERT INTO guild_twitch_subscriptions
87+
(guild_id, twitch_channel_id, notification_channel_id, notification_role_id, is_dm)
88+
VALUES ($1, $2, $3, $4, $5)
89+
`,
90+
[
91+
row.guild_id,
92+
row.platform_user_id,
93+
row.guild_channel_id,
94+
row.guild_ping_role,
95+
is_dm,
96+
],
97+
);
98+
} else if (row.guild_platform === "youtube") {
99+
await client.query(
100+
`
101+
INSERT INTO guild_youtube_subscriptions
102+
(guild_id, youtube_channel_id, notification_channel_id, notification_role_id, is_dm, track_videos, track_shorts, track_streams)
103+
VALUES ($1, $2, $3, $4, $5, false, false, false)
104+
`,
105+
[
106+
row.guild_id,
107+
row.platform_user_id,
108+
row.guild_channel_id,
109+
row.guild_ping_role,
110+
is_dm,
111+
],
112+
);
113+
}
114+
console.log(
115+
`➡️ Inserted ${row.guild_platform} subscription for guild ${row.guild_id} into subscriptions table`,
116+
);
117+
}
118+
119+
// Insert into bot_info
120+
if (botInfoData.length > 0) {
121+
const row = botInfoData[0] as any;
122+
123+
await client.query(
124+
`
125+
INSERT INTO bot_info (guilds_total, channels_tracked, total_members, updated_at, extended_info_updated_at)
126+
VALUES ($1, 0, $2, now(), now())
127+
`,
128+
[row.total_servers, row.total_members],
129+
);
130+
console.log(
131+
`➡️ Inserted bot info with guilds_total: ${row.total_servers} and total_members: ${row.total_members}`,
132+
);
133+
}
134+
135+
console.log("✅ Migration complete.");
136+
137+
// Sometimes the connection pool doesn't close properly
138+
// so we need to force it to close by... doing this
139+
process.exit(0);

‎src/utils/youtube/fetchLatestUploads.ts‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,34 @@
1+
import type { dbYouTube } from "../../types/database";
2+
13
import { ChannelType, TextChannel } from "discord.js";
24

35
import { env } from "../../config";
4-
import {
5-
getAllChannelsToTrack,
6-
getGuildsTrackingChannel,
7-
updateVideoId,
8-
} from "../database";
6+
import { getGuildsTrackingChannel, updateVideoId } from "../database";
97
import client from "../..";
8+
import { dbYouTubeGetAllChannelsToTrack } from "../db/youtube";
109

1110
import getChannelDetails from "./getChannelDetails";
1211

1312
export default async function fetchLatestUploads() {
1413
console.log("Fetching latest uploads...");
1514

16-
const channels = await getAllChannelsToTrack();
15+
const channels: dbYouTube[] | [] = await dbYouTubeGetAllChannelsToTrack();
1716
const channelDict: Record<string, string> = {};
1817

18+
if (!channels || channels.length === 0) {
19+
console.log("No channels to track.");
20+
21+
return;
22+
}
23+
1924
channels.forEach((channel) => {
25+
if (!channel.youtube_channel_id || !channel.latest_video_id) {
26+
console.error(
27+
"Channel ID or latest video ID is missing in fetchLatestUploads",
28+
);
29+
30+
return;
31+
}
2032
channelDict[channel.youtube_channel_id] = channel.latest_video_id;
2133
});
2234

@@ -98,7 +110,7 @@ export default async function fetchLatestUploads() {
98110
!channelObj ||
99111
(channelObj.type !== ChannelType.GuildText &&
100112
channelObj.type !==
101-
ChannelType.GuildAnnouncement)
113+
ChannelType.GuildAnnouncement)
102114
) {
103115
console.error(
104116
"Invalid channel or not a text channel in fetchLatestUploads",
@@ -111,10 +123,10 @@ export default async function fetchLatestUploads() {
111123
guild.guild_ping_role && channelInfo
112124
? `<@&${guild.guild_ping_role}> New video uploaded for ${channelInfo?.channelName}! https://www.youtube.com/watch?v=${videoId}`
113125
: guild.guild_ping_role
114-
? `<@&${guild.guild_ping_role}> New video uploaded! https://www.youtube.com/watch?v=${videoId}`
115-
: channelInfo
116-
? `New video uploaded for ${channelInfo.channelName}! https://www.youtube.com/watch?v=${videoId}`
117-
: `New video uploaded! https://www.youtube.com/watch?v=${videoId}`,
126+
? `<@&${guild.guild_ping_role}> New video uploaded! https://www.youtube.com/watch?v=${videoId}`
127+
: channelInfo
128+
? `New video uploaded for ${channelInfo.channelName}! https://www.youtube.com/watch?v=${videoId}`
129+
: `New video uploaded! https://www.youtube.com/watch?v=${videoId}`,
118130
});
119131
} catch (error) {
120132
console.error(

0 commit comments

Comments
 (0)