hub: cleanup data of deleted projects

Author: haraldschilly

src/packages/database/postgres-server-queries.coffee

@@ -51,7 +51,7 @@ read = require('read')
 {site_license_manager_set} = require('./postgres/site-license/manager')
 {matching_site_licenses, manager_site_licenses} = require('./postgres/site-license/search')
 {project_datastore_set, project_datastore_get, project_datastore_del} = require('./postgres/project-queries')
-{permanently_unlink_all_deleted_projects_of_user, unlink_old_deleted_projects} = require('./postgres/delete-projects')
+{permanently_unlink_all_deleted_projects_of_user, unlink_old_deleted_projects, cleanup_old_projects_data} = require('./postgres/delete-projects')
 {get_all_public_paths, unlist_all_public_paths} = require('./postgres/public-paths')
 {get_personal_user} = require('./postgres/personal')
 {set_passport_settings, get_passport_settings, get_all_passport_settings, get_all_passport_settings_cached, create_passport, passport_exists, update_account_and_passport, _passport_key} = require('./postgres/passport')
@@ -2590,6 +2590,10 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
     unlink_old_deleted_projects: () =>
         return await unlink_old_deleted_projects(@)
 
+    # async function
+    cleanup_old_projects_data: () =>
+        return await cleanup_old_projects_data(@)
+
     # async function
     unlist_all_public_paths: (account_id, is_owner) =>
         return await unlist_all_public_paths(@, account_id, is_owner)

src/packages/database/postgres/delete-projects.ts

@@ -7,8 +7,15 @@
 Code related to permanently deleting projects.
 */
 
+import getLogger from "@cocalc/backend/logger";
+import getPool from "@cocalc/database/pool";
 import { callback2 } from "@cocalc/util/async-utils";
 import { PostgreSQL } from "./types";
+import { minutes_ago } from "@cocalc/util/misc";
+import { getServerSettings } from "@cocalc/database/settings";
+import { KUCALC_ON_PREMISES } from "@cocalc/util/db-schema/site-defaults";
+
+const log = getLogger("db:delete-projects");
 
 /*
 Permanently delete from the database all project records, where the
@@ -20,7 +27,7 @@ later then purges these projects from disk as well as the database.
 */
 export async function permanently_unlink_all_deleted_projects_of_user(
   db: PostgreSQL,
-  account_id_or_email_address: string
+  account_id_or_email_address: string,
 ): Promise<void> {
   // Get the account_id if necessary.
   const account_id = await get_account_id(db, account_id_or_email_address);
@@ -36,7 +43,7 @@ export async function permanently_unlink_all_deleted_projects_of_user(
 
 async function get_account_id(
   db: PostgreSQL,
-  account_id_or_email_address: string
+  account_id_or_email_address: string,
 ): Promise<string> {
   if (account_id_or_email_address.indexOf("@") == -1) {
     return account_id_or_email_address;
@@ -57,7 +64,7 @@ Another task has to run to actually get rid of the data, etc.
 */
 export async function unlink_old_deleted_projects(
   db: PostgreSQL,
-  age_d = 30
+  age_d = 30,
 ): Promise<void> {
   await callback2(db._query, {
     query: "UPDATE projects",
@@ -69,3 +76,77 @@ export async function unlink_old_deleted_projects(
     ],
   });
 }
+
+const Q_CLEANUP_SYNCSTRINGS = `
+SELECT p.project_id, s.string_id
+FROM projects as p
+  INNER JOIN syncstrings as s
+  ON p.project_id = s.project_id
+WHERE p.deleted = true
+  AND p.state ->> 'state' != 'deleted'
+`;
+
+/*
+ This is more thorough than the above. It issues actual delete operations on data of projects marked as deleted.
+ When done, it sets the state.state to "deleted".
+
+ The operations involves deleting all syncstrings of that project (and associated with that, patches),
+ and only for on-prem setups, it also deletes all the data stored in the project on disk.
+
+ This function is called every couple of hours. Hence ensure it does not run longer than the given max_run_m time (minutes)
+*/
+export async function cleanup_old_projects_data(
+  db: PostgreSQL,
+  delay_ms = 50,
+  max_run_m = 60,
+) {
+  const settings = await getServerSettings();
+  const on_prem = settings.kucalc === KUCALC_ON_PREMISES;
+
+  log.debug("cleanup_old_projects_data", { delay_ms, max_run_m, on_prem });
+  const start_ts = new Date();
+
+  const pool = getPool();
+  const { rows } = await pool.query(Q_CLEANUP_SYNCSTRINGS);
+
+  let num = 0;
+  let pid = "";
+
+  for (const row of rows) {
+    const { project_id, string_id } = row;
+    if (start_ts < minutes_ago(max_run_m)) {
+      log.debug(
+        `cleanup_old_projects_data: too much time elapsed, breaking after ${num} syncstrings`,
+      );
+      break;
+    }
+
+    log.debug(
+      `cleanup_old_projects_data: deleting syncstring ${project_id}/${string_id}`,
+    );
+    num += 1;
+    await callback2(db.delete_syncstring, { string_id });
+
+    // wait for the given amount of delay_ms millio seconds
+    await new Promise((done) => setTimeout(done, delay_ms));
+
+    if (pid != project_id) {
+      pid = project_id;
+      if (on_prem) {
+        log.debug(
+          `cleanup_old_projects_data: deleting project data in ${project_id}`,
+        );
+        // TODO: this only works on-prem, and requires the project files to be mounted
+
+        log.debug(`deleting all shared files in project ${project_id}`);
+        // TODO: do it directly like above, and also get rid of all those shares in the database
+      }
+
+      // now, that we're done with that project, mark it as state.state ->> 'deleted'
+      await callback2(db.set_project_state, {
+        project_id,
+        state: "deleted",
+      });
+    }
+  }
+}

src/packages/database/postgres/types.ts

@@ -7,6 +7,7 @@ import { EventEmitter } from "events";
 import { Client } from "pg";
 
 import { PassportStrategyDB } from "@cocalc/database/settings/auth-sso-types";
+import { ProjectState } from "@cocalc/util/db-schema/projects";
 import {
   CB,
   CBDB,
@@ -305,6 +306,8 @@ export interface PostgreSQL extends EventEmitter {
     cb: CB;
   });
 
+  delete_syncstring(opts: { string_id: string; cb: CB });
+
   projects_that_need_to_be_started(): Promise<string[]>;
 
   is_connected(): boolean;
@@ -316,4 +319,6 @@ export interface PostgreSQL extends EventEmitter {
       email_address: string;
     }>;
   }): Promise<void>;
+
+  set_project_state(opts: { project_id: string; state: ProjectState["state"] });
 }

src/packages/hub/run/delete-projects.js

@@ -16,6 +16,7 @@ async function update() {
   console.log("unlinking old deleted projects...");
   try {
     await db.unlink_old_deleted_projects();
+    await db.cleanup_old_projects_data();
   } catch (err) {
     if (err !== null) {
       throw Error(`failed to unlink projects -- ${err}`);

src/packages/util/compute-states.ts

@@ -28,7 +28,8 @@ export type State =
   | "running"
   | "starting"
   | "stopping"
-  | "unarchiving";
+  | "unarchiving"
+  | "deleted";
 
 // @hsy: completely unclear what this is for.
 type Operation =
@@ -218,4 +219,14 @@ export const COMPUTE_STATES: ComputeState = {
       "migrate_live",
     ],
   },
+
+  // projects are deleted in hub -> postgres.delete-projects and this is a one-way operation
+  deleted: {
+    desc: "Project is deleted",
+    icon: "trash",
+    display: "Deleted",
+    stable: true,
+    to: {},
+    commands: [],
+  },
 } as const;