diff --git a/docs/content/1.getting-started/1.index.md b/docs/content/1.getting-started/1.index.md index 567cd9f..3311166 100644 --- a/docs/content/1.getting-started/1.index.md +++ b/docs/content/1.getting-started/1.index.md @@ -102,11 +102,15 @@ variant: subtle --- :: -#### `CACHE_CLEANUP_OLDER_THAN_DAYS` +#### `CACHE_CLEANUP_UNUSED_TTL_DAYS` -- Default: `90` +- Default: `30` -The number of days to keep stale cache data and metadata before deleting it. Set to `0` to disable cache cleanup. +Cache entries which have not been accessed for `CACHE_CLEANUP_UNUSED_TTL_DAYS` days will be automatically deleted. Set to `0` to disable. + +#### `CACHE_CLEANUP_TTL_DAYS` + +Cache entries which have been created `CACHE_CLEANUP_TTL_DAYS` days ago will be automatically deleted, regardless of when it was last accessed. #### `CACHE_CLEANUP_CRON` diff --git a/lib/db/index.ts b/lib/db/index.ts index 2f949a5..65a7ca7 100644 --- a/lib/db/index.ts +++ b/lib/db/index.ts @@ -18,6 +18,7 @@ export interface CacheKeysTable { version: string updated_at: string accessed_at: string + created_at: string } export interface UploadsTable { created_at: string @@ -191,17 +192,26 @@ export async function touchKey( export async function findStaleKeys( db: DB, - { olderThanDays, date }: { olderThanDays?: number; date?: Date }, + opts?: { + unusedTTLDays?: number + ttlDays?: number + date?: Date + }, ) { - if (olderThanDays === undefined) return db.selectFrom('cache_keys').selectAll().execute() + const now = opts?.date ?? new Date() + let query = db.selectFrom('cache_keys') - const now = date ?? new Date() - const threshold = new Date(now.getTime() - olderThanDays * 24 * 60 * 60 * 1000) - return db - .selectFrom('cache_keys') - .where('accessed_at', '<', threshold.toISOString()) - .selectAll() - .execute() + if (opts?.ttlDays !== undefined) { + const threshold = new Date(now.getTime() - opts.ttlDays * 24 * 60 * 60 * 1000) + query = query.where('created_at', '<', threshold.toISOString()) + } + + if (opts?.unusedTTLDays !== undefined) { + const threshold = new Date(now.getTime() - opts.unusedTTLDays * 24 * 60 * 60 * 1000) + query = query.where('accessed_at', '<', threshold.toISOString()) + } + + return query.selectAll().execute() } export async function createKey( @@ -217,6 +227,7 @@ export async function createKey( version, updated_at: now.toISOString(), accessed_at: now.toISOString(), + created_at: now.toISOString(), }) .execute() } diff --git a/lib/db/migrations.ts b/lib/db/migrations.ts index f23e950..2971165 100644 --- a/lib/db/migrations.ts +++ b/lib/db/migrations.ts @@ -1,23 +1,33 @@ -import type { Migration } from 'kysely' +import type { Kysely, Migration } from 'kysely' import type { DatabaseDriverName } from '~/lib/db/drivers' import { sql } from 'kysely' +async function createCacheKeysTable({ + db, + dbType, +}: { + db: Kysely + dbType: DatabaseDriverName +}) { + let query = db.schema + .createTable('cache_keys') + .addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey()) + .addColumn('key', 'text', (col) => col.notNull()) + .addColumn('version', 'text', (col) => col.notNull()) + .addColumn('updated_at', 'text', (col) => col.notNull()) + .addColumn('accessed_at', 'text', (col) => col.notNull()) + + if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`) + + await query.ifNotExists().execute() +} + export function migrations(dbType: DatabaseDriverName) { return { $0_cache_keys_table: { async up(db) { - let query = db.schema - .createTable('cache_keys') - .addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey()) - .addColumn('key', 'text', (col) => col.notNull()) - .addColumn('version', 'text', (col) => col.notNull()) - .addColumn('updated_at', 'text', (col) => col.notNull()) - .addColumn('accessed_at', 'text', (col) => col.notNull()) - - if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`) - - await query.ifNotExists().execute() + await createCacheKeysTable({ db, dbType }) }, async down(db) { await db.schema.dropTable('cache_keys').ifExists().execute() @@ -76,5 +86,48 @@ export function migrations(dbType: DatabaseDriverName) { await db.schema.alterTable('upload_parts').dropColumn('e_tag').execute() }, }, + $4_cache_entry_created_at: { + async up(db) { + await db.schema.alterTable('cache_keys').addColumn('created_at', 'text').execute() + + await db + .updateTable('cache_keys') + .set({ + created_at: new Date().toISOString(), + }) + .execute() + + if (dbType === 'mysql') + await db.schema + .alterTable('cache_keys') + .modifyColumn('created_at', 'text', (c) => c.notNull()) + .execute() + else if (dbType === 'postgres') + await db.schema + .alterTable('cache_keys') + .alterColumn('created_at', (c) => c.setNotNull()) + .execute() + else { + // rename old table + await db.schema.alterTable('cache_keys').renameTo('old_cache_keys').execute() + // recreate table + await createCacheKeysTable({ db, dbType }) + + // add not null column + await db.schema + .alterTable('cache_keys') + .addColumn('created_at', 'text', (c) => c.notNull()) + .execute() + + // migrate old data + await db + .insertInto('cache_keys') + .expression((e) => e.selectFrom('old_cache_keys').selectAll()) + .execute() + + await db.schema.dropTable('old_cache_keys').execute() + } + }, + }, } satisfies Record } diff --git a/lib/env.ts b/lib/env.ts index da5f665..0f0d0f7 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -6,7 +6,12 @@ const portSchema = z.coerce.number().int().min(1).max(65_535) const envSchema = z.object({ ENABLE_DIRECT_DOWNLOADS: booleanSchema.default('false'), + /** + * @deprecated use `CACHE_CLEANUP_UNUSED_TTL_DAYS` or `CACHE_CLEANUP_TTL_DAYS` instead + */ CACHE_CLEANUP_OLDER_THAN_DAYS: z.coerce.number().int().min(0).default(90), + CACHE_CLEANUP_UNUSED_TTL_DAYS: z.coerce.number().int().min(0).default(30), + CACHE_CLEANUP_TTL_DAYS: z.coerce.number().int().min(0).optional(), CACHE_CLEANUP_CRON: z.string().default('0 0 * * *'), UPLOAD_CLEANUP_CRON: z.string().default('*/10 * * * *'), API_BASE_URL: z.string().url(), diff --git a/lib/storage/index.ts b/lib/storage/index.ts index d759d9b..62d864a 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -184,12 +184,10 @@ export const useStorageAdapter = createSingletonPromise(async () => { logger.debug('Download:', objectName) return driver.createReadStream(objectName) }, - async pruneCaches(olderThanDays?: number) { - logger.debug('Prune:', { - olderThanDays, - }) + async pruneCaches(opts?: { unusedTTLDays?: number; ttlDays?: number }) { + logger.debug('Prune:', opts) - const keys = await findStaleKeys(db, { olderThanDays }) + const keys = await findStaleKeys(db, opts) if (keys.length === 0) { logger.debug('Prune: No caches to prune') return @@ -198,9 +196,7 @@ export const useStorageAdapter = createSingletonPromise(async () => { await driver.delete(keys.map((key) => getObjectNameFromKey(key.key, key.version))) await pruneKeys(db, keys) - logger.debug('Prune: Caches pruned', { - olderThanDays, - }) + logger.debug('Prune: Caches pruned', opts) }, async pruneUploads(olderThanDate: Date) { logger.debug('Prune uploads') diff --git a/plugins/cleanup.ts b/plugins/cleanup.ts index cdd885c..f6a725a 100644 --- a/plugins/cleanup.ts +++ b/plugins/cleanup.ts @@ -11,15 +11,22 @@ export default defineNitroPlugin(() => { if (!cluster.isPrimary) return // cache cleanup - if (ENV.CACHE_CLEANUP_OLDER_THAN_DAYS > 0) { + if ( + ENV.CACHE_CLEANUP_OLDER_THAN_DAYS || + ENV.CACHE_CLEANUP_TTL_DAYS || + ENV.CACHE_CLEANUP_UNUSED_TTL_DAYS + ) { const job = new Cron(ENV.CACHE_CLEANUP_CRON) const nextRun = job.nextRun() logger.info( - `Cleaning up cache entries older than ${colorize('blue', `${ENV.CACHE_CLEANUP_OLDER_THAN_DAYS}d`)} with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`, + `Cleaning up cache entries with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`, ) job.schedule(async () => { const adapter = await useStorageAdapter() - await adapter.pruneCaches(ENV.CACHE_CLEANUP_OLDER_THAN_DAYS) + await adapter.pruneCaches({ + ttlDays: ENV.CACHE_CLEANUP_TTL_DAYS, + unusedTTLDays: ENV.CACHE_CLEANUP_UNUSED_TTL_DAYS ?? ENV.CACHE_CLEANUP_OLDER_THAN_DAYS, + }) }) } diff --git a/tests/cleanup.test.ts b/tests/cleanup.test.ts index cf21990..840420e 100644 --- a/tests/cleanup.test.ts +++ b/tests/cleanup.test.ts @@ -61,14 +61,14 @@ describe('getting stale keys', async () => { beforeEach(() => pruneKeys(db)) const version = '0577ec58bee6d5415625' - test('returns stale keys if threshold is passed', async () => { + test('returns unused stale keys if threshold is passed', async () => { const referenceDate = new Date('2024-04-01T00:00:00Z') await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') }) await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') }) - const match = await findStaleKeys(db, { olderThanDays: 30, date: referenceDate }) + const match = await findStaleKeys(db, { unusedTTLDays: 30, date: referenceDate }) expect(match.length).toBe(2) const matchA = match.find((m) => m.key === 'cache-a') @@ -80,6 +80,29 @@ describe('getting stale keys', async () => { expect(matchB?.accessed_at).toBe('2024-02-01T00:00:00.000Z') }) + test('returns stale keys if threshold is passed', async () => { + const referenceDate = new Date('2024-04-01T00:00:00Z') + await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') }) + await touchKey(db, { key: 'cache-a', version, date: referenceDate }) + await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') }) + await touchKey(db, { key: 'cache-b', version, date: referenceDate }) + await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') }) + await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') }) + + const match = await findStaleKeys(db, { ttlDays: 30, date: referenceDate }) + expect(match.length).toBe(2) + + const matchA = match.find((m) => m.key === 'cache-a') + expect(matchA).toBeDefined() + expect(matchA?.created_at).toBe('2024-01-01T00:00:00.000Z') + expect(matchA?.accessed_at).toBe('2024-04-01T00:00:00.000Z') + + const matchB = match.find((m) => m.key === 'cache-b') + expect(matchB).toBeDefined() + expect(matchB?.created_at).toBe('2024-02-01T00:00:00.000Z') + expect(matchB?.accessed_at).toBe('2024-04-01T00:00:00.000Z') + }) + test('returns all keys if threshold is not passed', async () => { const referenceDate = new Date('2024-04-01T00:00:00Z') await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') })