From a5e47c4b770a8ac328200608beeaa57117a9bd93 Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:53:57 +0200 Subject: [PATCH 1/3] feat: improve cache cleanup --- docs/content/1.getting-started/1.index.md | 10 ++- lib/db/index.ts | 29 +++++--- lib/db/migrations.ts | 87 +++++++++++++++++++---- lib/env.ts | 5 ++ lib/storage/index.ts | 12 ++-- plugins/cleanup.ts | 13 +++- tests/cleanup.test.ts | 27 ++++++- 7 files changed, 146 insertions(+), 37 deletions(-) diff --git a/docs/content/1.getting-started/1.index.md b/docs/content/1.getting-started/1.index.md index 567cd9f..a94e846 100644 --- a/docs/content/1.getting-started/1.index.md +++ b/docs/content/1.getting-started/1.index.md @@ -102,11 +102,15 @@ variant: subtle --- :: -#### `CACHE_CLEANUP_OLDER_THAN_DAYS` +#### `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` -- Default: `90` +- Default: `30` -The number of days to keep stale cache data and metadata before deleting it. Set to `0` to disable cache cleanup. +Cache entries which have not been accessed for `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` days will be automatically deleted. Set to `0` to disable. + +#### `CACHE_CLEANUP_TTL_DAYS` + +Cache entries which have been created `CACHE_CLEANUP_TTL_DAYS` days ago will be automatically deleted. Set to `0` to disable. #### `CACHE_CLEANUP_CRON` diff --git a/lib/db/index.ts b/lib/db/index.ts index 2f949a5..0bd4ede 100644 --- a/lib/db/index.ts +++ b/lib/db/index.ts @@ -18,6 +18,7 @@ export interface CacheKeysTable { version: string updated_at: string accessed_at: string + created_at: string } export interface UploadsTable { created_at: string @@ -191,17 +192,26 @@ export async function touchKey( export async function findStaleKeys( db: DB, - { olderThanDays, date }: { olderThanDays?: number; date?: Date }, + opts?: { + untouchedTTLDays?: number + ttlDays?: number + date?: Date + }, ) { - if (olderThanDays === undefined) return db.selectFrom('cache_keys').selectAll().execute() + const now = opts?.date ?? new Date() + let query = db.selectFrom('cache_keys') - const now = date ?? new Date() - const threshold = new Date(now.getTime() - olderThanDays * 24 * 60 * 60 * 1000) - return db - .selectFrom('cache_keys') - .where('accessed_at', '<', threshold.toISOString()) - .selectAll() - .execute() + if (opts?.ttlDays !== undefined) { + const threshold = new Date(now.getTime() - opts.ttlDays * 24 * 60 * 60 * 1000) + query = query.where('created_at', '<', threshold.toISOString()) + } + + if (opts?.untouchedTTLDays !== undefined) { + const untouchedThreshold = new Date(now.getTime() - opts.untouchedTTLDays * 24 * 60 * 60 * 1000) + query = query.where('accessed_at', '<', untouchedThreshold.toISOString()) + } + + return query.selectAll().execute() } export async function createKey( @@ -217,6 +227,7 @@ export async function createKey( version, updated_at: now.toISOString(), accessed_at: now.toISOString(), + created_at: now.toISOString(), }) .execute() } diff --git a/lib/db/migrations.ts b/lib/db/migrations.ts index f23e950..56bbe91 100644 --- a/lib/db/migrations.ts +++ b/lib/db/migrations.ts @@ -1,23 +1,33 @@ -import type { Migration } from 'kysely' +import type { Kysely, Migration } from 'kysely' import type { DatabaseDriverName } from '~/lib/db/drivers' import { sql } from 'kysely' +async function createCacheKeysTable({ + db, + dbType, +}: { + db: Kysely + dbType: DatabaseDriverName +}) { + let query = db.schema + .createTable('cache_keys') + .addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey()) + .addColumn('key', 'text', (col) => col.notNull()) + .addColumn('version', 'text', (col) => col.notNull()) + .addColumn('updated_at', 'text', (col) => col.notNull()) + .addColumn('accessed_at', 'text', (col) => col.notNull()) + + if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`) + + await query.ifNotExists().execute() +} + export function migrations(dbType: DatabaseDriverName) { return { $0_cache_keys_table: { async up(db) { - let query = db.schema - .createTable('cache_keys') - .addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey()) - .addColumn('key', 'text', (col) => col.notNull()) - .addColumn('version', 'text', (col) => col.notNull()) - .addColumn('updated_at', 'text', (col) => col.notNull()) - .addColumn('accessed_at', 'text', (col) => col.notNull()) - - if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`) - - await query.ifNotExists().execute() + await createCacheKeysTable({ db, dbType }) }, async down(db) { await db.schema.dropTable('cache_keys').ifExists().execute() @@ -76,5 +86,58 @@ export function migrations(dbType: DatabaseDriverName) { await db.schema.alterTable('upload_parts').dropColumn('e_tag').execute() }, }, + $4_cache_entry_created_at: { + async up(db) { + await db + .insertInto('cache_keys') + .values({ + id: '', + key: '', + version: '', + updated_at: new Date().toISOString(), + accessed_at: new Date().toISOString(), + }) + .execute() + await db.schema.alterTable('cache_keys').addColumn('created_at', 'text').execute() + + await db + .updateTable('cache_keys') + .set({ + created_at: new Date().toISOString(), + }) + .execute() + + if (dbType === 'mysql') + await db.schema + .alterTable('cache_keys') + .modifyColumn('created_at', 'text', (c) => c.notNull()) + .execute() + else if (dbType === 'postgres') + await db.schema + .alterTable('cache_keys') + .alterColumn('created_at', (c) => c.setNotNull()) + .execute() + else { + // rename old table + await db.schema.alterTable('cache_keys').renameTo('old_cache_keys').execute() + // recreate table + await createCacheKeysTable({ db, dbType }) + + // add not null column + await db.schema + .alterTable('cache_keys') + .addColumn('created_at', 'text', (c) => c.notNull()) + .execute() + + // migrate old data + await db + .insertInto('cache_keys') + .expression((e) => e.selectFrom('old_cache_keys').selectAll()) + .execute() + + await db.schema.dropTable('old_cache_keys').execute() + } + }, + }, } satisfies Record } diff --git a/lib/env.ts b/lib/env.ts index da5f665..c3c9a6a 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -6,7 +6,12 @@ const portSchema = z.coerce.number().int().min(1).max(65_535) const envSchema = z.object({ ENABLE_DIRECT_DOWNLOADS: booleanSchema.default('false'), + /** + * @deprecated use `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` or `CACHE_CLEANUP_TTL_DAYS` instead + */ CACHE_CLEANUP_OLDER_THAN_DAYS: z.coerce.number().int().min(0).default(90), + CACHE_CLEANUP_UNTOUCHED_TTL_DAYS: z.coerce.number().int().min(0).default(30), + CACHE_CLEANUP_TTL_DAYS: z.coerce.number().int().min(0).optional(), CACHE_CLEANUP_CRON: z.string().default('0 0 * * *'), UPLOAD_CLEANUP_CRON: z.string().default('*/10 * * * *'), API_BASE_URL: z.string().url(), diff --git a/lib/storage/index.ts b/lib/storage/index.ts index d759d9b..3c5ebaa 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -184,12 +184,10 @@ export const useStorageAdapter = createSingletonPromise(async () => { logger.debug('Download:', objectName) return driver.createReadStream(objectName) }, - async pruneCaches(olderThanDays?: number) { - logger.debug('Prune:', { - olderThanDays, - }) + async pruneCaches(opts?: { untouchedTTLDays?: number; ttlDays?: number }) { + logger.debug('Prune:', opts) - const keys = await findStaleKeys(db, { olderThanDays }) + const keys = await findStaleKeys(db, opts) if (keys.length === 0) { logger.debug('Prune: No caches to prune') return @@ -198,9 +196,7 @@ export const useStorageAdapter = createSingletonPromise(async () => { await driver.delete(keys.map((key) => getObjectNameFromKey(key.key, key.version))) await pruneKeys(db, keys) - logger.debug('Prune: Caches pruned', { - olderThanDays, - }) + logger.debug('Prune: Caches pruned', opts) }, async pruneUploads(olderThanDate: Date) { logger.debug('Prune uploads') diff --git a/plugins/cleanup.ts b/plugins/cleanup.ts index cdd885c..2f4f937 100644 --- a/plugins/cleanup.ts +++ b/plugins/cleanup.ts @@ -11,15 +11,22 @@ export default defineNitroPlugin(() => { if (!cluster.isPrimary) return // cache cleanup - if (ENV.CACHE_CLEANUP_OLDER_THAN_DAYS > 0) { + if ( + ENV.CACHE_CLEANUP_OLDER_THAN_DAYS || + ENV.CACHE_CLEANUP_TTL_DAYS || + ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS + ) { const job = new Cron(ENV.CACHE_CLEANUP_CRON) const nextRun = job.nextRun() logger.info( - `Cleaning up cache entries older than ${colorize('blue', `${ENV.CACHE_CLEANUP_OLDER_THAN_DAYS}d`)} with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`, + `Cleaning up cache entries with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`, ) job.schedule(async () => { const adapter = await useStorageAdapter() - await adapter.pruneCaches(ENV.CACHE_CLEANUP_OLDER_THAN_DAYS) + await adapter.pruneCaches({ + ttlDays: ENV.CACHE_CLEANUP_TTL_DAYS, + untouchedTTLDays: ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS ?? ENV.CACHE_CLEANUP_OLDER_THAN_DAYS, + }) }) } diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index cf21990..4482ffc 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -61,14 +61,14 @@ describe('getting stale keys', async () => { beforeEach(() => pruneKeys(db)) const version = '0577ec58bee6d5415625' - test('returns stale keys if threshold is passed', async () => { + test('returns untouched stale keys if threshold is passed', async () => { const referenceDate = new Date('2024-04-01T00:00:00Z') await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') }) - const match = await findStaleKeys(db, { olderThanDays: 30, date: referenceDate }) + const match = await findStaleKeys(db, { untouchedTTLDays: 30, date: referenceDate }) expect(match.length).toBe(2) const matchA = match.find((m) => m.key === 'cache-a') @@ -80,6 +80,29 @@ describe('getting stale keys', async () => { expect(matchB?.accessed_at).toBe('2024-02-01T00:00:00.000Z') }) + test('returns stale keys if threshold is passed', async () => { + const referenceDate = new Date('2024-04-01T00:00:00Z') + await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') }) + await touchKey(db, { key: 'cache-a', version, date: referenceDate }) + await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') }) + await touchKey(db, { key: 'cache-b', version, date: referenceDate }) + await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') }) + await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') }) + + const match = await findStaleKeys(db, { ttlDays: 30, date: referenceDate }) + expect(match.length).toBe(2) + + const matchA = match.find((m) => m.key === 'cache-a') + expect(matchA).toBeDefined() + expect(matchA?.created_at).toBe('2024-01-01T00:00:00.000Z') + expect(matchA?.accessed_at).toBe('2024-04-01T00:00:00.000Z') + + const matchB = match.find((m) => m.key === 'cache-b') + expect(matchB).toBeDefined() + expect(matchB?.created_at).toBe('2024-02-01T00:00:00.000Z') + expect(matchB?.accessed_at).toBe('2024-04-01T00:00:00.000Z') + }) + test('returns all keys if threshold is not passed', async () => { const referenceDate = new Date('2024-04-01T00:00:00Z') await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') }) From e1d7a014acdafaf46f46300cc419418dc0854bca Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Tue, 1 Jul 2025 14:46:32 +0200 Subject: [PATCH 2/3] fix: migration --- lib/db/migrations.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/db/migrations.ts b/lib/db/migrations.ts index 56bbe91..2971165 100644 --- a/lib/db/migrations.ts +++ b/lib/db/migrations.ts @@ -88,16 +88,6 @@ export function migrations(dbType: DatabaseDriverName) { }, $4_cache_entry_created_at: { async up(db) { - await db - .insertInto('cache_keys') - .values({ - id: '', - key: '', - version: '', - updated_at: new Date().toISOString(), - accessed_at: new Date().toISOString(), - }) - .execute() await db.schema.alterTable('cache_keys').addColumn('created_at', 'text').execute() await db From 7c1f3b23e2111ee2aa57bed142d1b169616829e7 Mon Sep 17 00:00:00 2001 From: Louis Haftmann <30736553+LouisHaftmann@users.noreply.github.com> Date: Tue, 1 Jul 2025 15:28:33 +0200 Subject: [PATCH 3/3] refactor: wording --- docs/content/1.getting-started/1.index.md | 6 +++--- lib/db/index.ts | 8 ++++---- lib/env.ts | 4 ++-- lib/storage/index.ts | 2 +- plugins/cleanup.ts | 4 ++-- tests/cleanup.test.ts | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/content/1.getting-started/1.index.md b/docs/content/1.getting-started/1.index.md index a94e846..3311166 100644 --- a/docs/content/1.getting-started/1.index.md +++ b/docs/content/1.getting-started/1.index.md @@ -102,15 +102,15 @@ variant: subtle --- :: -#### `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` +#### `CACHE_CLEANUP_UNUSED_TTL_DAYS` - Default: `30` -Cache entries which have not been accessed for `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` days will be automatically deleted. Set to `0` to disable. +Cache entries which have not been accessed for `CACHE_CLEANUP_UNUSED_TTL_DAYS` days will be automatically deleted. Set to `0` to disable. #### `CACHE_CLEANUP_TTL_DAYS` -Cache entries which have been created `CACHE_CLEANUP_TTL_DAYS` days ago will be automatically deleted. Set to `0` to disable. +Cache entries which have been created `CACHE_CLEANUP_TTL_DAYS` days ago will be automatically deleted, regardless of when it was last accessed. #### `CACHE_CLEANUP_CRON` diff --git a/lib/db/index.ts b/lib/db/index.ts index 0bd4ede..65a7ca7 100644 --- a/lib/db/index.ts +++ b/lib/db/index.ts @@ -193,7 +193,7 @@ export async function touchKey( export async function findStaleKeys( db: DB, opts?: { - untouchedTTLDays?: number + unusedTTLDays?: number ttlDays?: number date?: Date }, @@ -206,9 +206,9 @@ export async function findStaleKeys( query = query.where('created_at', '<', threshold.toISOString()) } - if (opts?.untouchedTTLDays !== undefined) { - const untouchedThreshold = new Date(now.getTime() - opts.untouchedTTLDays * 24 * 60 * 60 * 1000) - query = query.where('accessed_at', '<', untouchedThreshold.toISOString()) + if (opts?.unusedTTLDays !== undefined) { + const threshold = new Date(now.getTime() - opts.unusedTTLDays * 24 * 60 * 60 * 1000) + query = query.where('accessed_at', '<', threshold.toISOString()) } return query.selectAll().execute() diff --git a/lib/env.ts b/lib/env.ts index c3c9a6a..0f0d0f7 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -7,10 +7,10 @@ const portSchema = z.coerce.number().int().min(1).max(65_535) const envSchema = z.object({ ENABLE_DIRECT_DOWNLOADS: booleanSchema.default('false'), /** - * @deprecated use `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` or `CACHE_CLEANUP_TTL_DAYS` instead + * @deprecated use `CACHE_CLEANUP_UNUSED_TTL_DAYS` or `CACHE_CLEANUP_TTL_DAYS` instead */ CACHE_CLEANUP_OLDER_THAN_DAYS: z.coerce.number().int().min(0).default(90), - CACHE_CLEANUP_UNTOUCHED_TTL_DAYS: z.coerce.number().int().min(0).default(30), + CACHE_CLEANUP_UNUSED_TTL_DAYS: z.coerce.number().int().min(0).default(30), CACHE_CLEANUP_TTL_DAYS: z.coerce.number().int().min(0).optional(), CACHE_CLEANUP_CRON: z.string().default('0 0 * * *'), UPLOAD_CLEANUP_CRON: z.string().default('*/10 * * * *'), diff --git a/lib/storage/index.ts b/lib/storage/index.ts index 3c5ebaa..62d864a 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -184,7 +184,7 @@ export const useStorageAdapter = createSingletonPromise(async () => { logger.debug('Download:', objectName) return driver.createReadStream(objectName) }, - async pruneCaches(opts?: { untouchedTTLDays?: number; ttlDays?: number }) { + async pruneCaches(opts?: { unusedTTLDays?: number; ttlDays?: number }) { logger.debug('Prune:', opts) const keys = await findStaleKeys(db, opts) diff --git a/plugins/cleanup.ts b/plugins/cleanup.ts index 2f4f937..f6a725a 100644 --- a/plugins/cleanup.ts +++ b/plugins/cleanup.ts @@ -14,7 +14,7 @@ export default defineNitroPlugin(() => { if ( ENV.CACHE_CLEANUP_OLDER_THAN_DAYS || ENV.CACHE_CLEANUP_TTL_DAYS || - ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS + ENV.CACHE_CLEANUP_UNUSED_TTL_DAYS ) { const job = new Cron(ENV.CACHE_CLEANUP_CRON) const nextRun = job.nextRun() @@ -25,7 +25,7 @@ export default defineNitroPlugin(() => { const adapter = await useStorageAdapter() await adapter.pruneCaches({ ttlDays: ENV.CACHE_CLEANUP_TTL_DAYS, - untouchedTTLDays: ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS ?? ENV.CACHE_CLEANUP_OLDER_THAN_DAYS, + unusedTTLDays: ENV.CACHE_CLEANUP_UNUSED_TTL_DAYS ?? ENV.CACHE_CLEANUP_OLDER_THAN_DAYS, }) }) } diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index 4482ffc..840420e 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -61,14 +61,14 @@ describe('getting stale keys', async () => { beforeEach(() => pruneKeys(db)) const version = '0577ec58bee6d5415625' - test('returns untouched stale keys if threshold is passed', async () => { + test('returns unused stale keys if threshold is passed', async () => { const referenceDate = new Date('2024-04-01T00:00:00Z') await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') }) - const match = await findStaleKeys(db, { untouchedTTLDays: 30, date: referenceDate }) + const match = await findStaleKeys(db, { unusedTTLDays: 30, date: referenceDate }) expect(match.length).toBe(2) const matchA = match.find((m) => m.key === 'cache-a')