feat(migrations): add configurable data retention TTL for self-hosted instances

.changeset/brave-mangos-visit.md

@@ -0,0 +1,15 @@
+---
+'@hive/migrations': minor
+---
+
+Add configurable data retention TTL for self-hosted Hive instances. Self-hosted users can now configure retention periods via environment variables instead of hardcoded values.
+
+New environment variables:
+- `CLICKHOUSE_TTL_TABLES` - Retention for ClickHouse mergetree tables (Default: 1 YEAR)
+- `CLICKHOUSE_TTL_DAILY_MV_TABLES` - Retention for daily materialized view tables (Default: 1 YEAR)
+- `CLICKHOUSE_TTL_HOURLY_MV_TABLES` - Retention for hourly materialized view tables (Default: 30 DAYS)
+- `CLICKHOUSE_TTL_MINUTELY_MV_TABLES` - Retention for minutely materialized view tables (Default: 24 HOURS)
+
+Supports both numeric days (e.g., `365`) and ClickHouse interval syntax (e.g., `"1 YEAR"`, `"30 DAY"`, `"24 HOUR"`).
+
+The retention update runs automatically if any retention environment variable is set.

docker/docker-compose.community.yml

@@ -170,6 +170,10 @@ services:
       CLICKHOUSE_PORT: '8123'
       CLICKHOUSE_USERNAME: '${CLICKHOUSE_USER}'
       CLICKHOUSE_PASSWORD: '${CLICKHOUSE_PASSWORD}'
+      CLICKHOUSE_TTL_TABLES: '${CLICKHOUSE_TTL_TABLES:-}'
+      CLICKHOUSE_TTL_DAILY_MV_TABLES: '${CLICKHOUSE_TTL_DAILY_MV_TABLES:-}'
+      CLICKHOUSE_TTL_HOURLY_MV_TABLES: '${CLICKHOUSE_TTL_HOURLY_MV_TABLES:-}'
+      CLICKHOUSE_TTL_MINUTELY_MV_TABLES: '${CLICKHOUSE_TTL_MINUTELY_MV_TABLES:-}'
       TS_NODE_TRANSPILE_ONLY: 'true'
       LOG_LEVEL: '${LOG_LEVEL:-debug}'
 

packages/migrations/.env.template

@@ -5,6 +5,10 @@ CLICKHOUSE_HOST="localhost"
 CLICKHOUSE_PORT="8123"
 CLICKHOUSE_USERNAME="test"
 CLICKHOUSE_PASSWORD="test"
+# CLICKHOUSE_TTL_TABLES="1 YEAR"
+# CLICKHOUSE_TTL_DAILY_MV_TABLES="30 DAYS"
+# CLICKHOUSE_TTL_HOURLY_MV_TABLES="7 DAYS"
+# CLICKHOUSE_TTL_MINUTELY_MV_TABLES="1 DAY"
 
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres

packages/migrations/src/index.ts

@@ -5,6 +5,7 @@ import { migrateClickHouse } from './clickhouse';
 import { createConnectionString } from './connection-string';
 import { env } from './environment';
 import { runPGMigrations } from './run-pg-migrations';
+import { updateRetention } from './scripts/update-retention';
 
 const slonik = await createPool(createConnectionString(env.postgres), {
   // 10 minute timeout per statement
@@ -38,6 +39,27 @@ try {
       env.clickhouse,
     );
   }
+
+  // Automatically apply retention if any retention setting is configured
+  // eslint-disable-next-line no-process-env
+  if (
+    // eslint-disable-next-line no-process-env
+    process.env.CLICKHOUSE_TTL_TABLES ||
+    // eslint-disable-next-line no-process-env
+    process.env.CLICKHOUSE_TTL_DAILY_MV_TABLES ||
+    // eslint-disable-next-line no-process-env
+    process.env.CLICKHOUSE_TTL_HOURLY_MV_TABLES ||
+    // eslint-disable-next-line no-process-env
+    process.env.CLICKHOUSE_TTL_MINUTELY_MV_TABLES
+  ) {
+    console.log('Applying clickhouse retention settings...');
+    try {
+      await updateRetention();
+    } catch (error) {
+      console.error('Failed to update retention (non-fatal):', error);
+    }
+  }
+
   process.exit(0);
 } catch (error) {
   console.error(error);