feat: implement hybrid cache architecture for repository cloning

fabiovincenzi · fabiovincenzi · commit 956f3883cfc6 · 2025-10-10T16:18:20.000+02:00
diff --git a/src/proxy/processors/push-action/cache-manager.ts b/src/proxy/processors/push-action/cache-manager.ts
@@ -0,0 +1,181 @@
+import fs from 'fs';
+import path from 'path';
+import { getCacheConfig } from '../../../config';
+
+export interface CacheStats {
+  totalRepositories: number;
+  totalSizeMB: number;
+  repositories: Array<{
+    name: string;
+    sizeMB: number;
+    lastAccessed: Date;
+  }>;
+}
+
+export class CacheManager {
+  private cacheDir: string;
+  private maxSizeGB: number;
+  private maxRepositories: number;
+
+  constructor(
+    cacheDir: string = './.remote/cache',
+    maxSizeGB: number = 2,
+    maxRepositories: number = 50,
+  ) {
+    this.cacheDir = cacheDir;
+    this.maxSizeGB = maxSizeGB;
+    this.maxRepositories = maxRepositories;
+  }
+
+  /**
+   * Update access time for repository (for LRU purposes)
+   */
+  touchRepository(repoName: string): void {
+    const repoPath = path.join(this.cacheDir, repoName);
+    if (fs.existsSync(repoPath)) {
+      const now = new Date();
+      fs.utimesSync(repoPath, now, now);
+    }
+  }
+
+  /**
+   * Get cache statistics
+   */
+  getCacheStats(): CacheStats {
+    if (!fs.existsSync(this.cacheDir)) {
+      return {
+        totalRepositories: 0,
+        totalSizeMB: 0,
+        repositories: [],
+      };
+    }
+
+    const repositories: Array<{ name: string; sizeMB: number; lastAccessed: Date }> = [];
+    let totalSizeMB = 0;
+
+    const entries = fs.readdirSync(this.cacheDir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        const repoPath = path.join(this.cacheDir, entry.name);
+        const sizeMB = this.getDirectorySize(repoPath);
+        const stats = fs.statSync(repoPath);
+
+        repositories.push({
+          name: entry.name,
+          sizeMB,
+          lastAccessed: stats.atime,
+        });
+
+        totalSizeMB += sizeMB;
+      }
+    }
+
+    // Sort by last accessed (newest first)
+    repositories.sort((a, b) => b.lastAccessed.getTime() - a.lastAccessed.getTime());
+
+    return {
+      totalRepositories: repositories.length,
+      totalSizeMB,
+      repositories,
+    };
+  }
+
+  /**
+   * Enforce cache limits using LRU eviction
+   */
+  enforceLimits(): { removedRepos: string[]; freedMB: number } {
+    const stats = this.getCacheStats();
+    const removedRepos: string[] = [];
+    let freedMB = 0;
+
+    // Sort repositories by last accessed (oldest first for removal)
+    const reposToEvaluate = [...stats.repositories].sort(
+      (a, b) => a.lastAccessed.getTime() - b.lastAccessed.getTime(),
+    );
+
+    // Check size limit
+    let currentSizeMB = stats.totalSizeMB;
+    const maxSizeMB = this.maxSizeGB * 1024;
+
+    for (const repo of reposToEvaluate) {
+      const shouldRemove =
+        currentSizeMB > maxSizeMB || // Over size limit
+        stats.totalRepositories - removedRepos.length > this.maxRepositories; // Over count limit
+
+      if (shouldRemove) {
+        this.removeRepository(repo.name);
+        removedRepos.push(repo.name);
+        freedMB += repo.sizeMB;
+        currentSizeMB -= repo.sizeMB;
+      } else {
+        break; // We've cleaned enough
+      }
+    }
+
+    return { removedRepos, freedMB };
+  }
+
+  /**
+   * Remove specific repository from cache
+   */
+  private removeRepository(repoName: string): void {
+    const repoPath = path.join(this.cacheDir, repoName);
+    if (fs.existsSync(repoPath)) {
+      fs.rmSync(repoPath, { recursive: true, force: true });
+    }
+  }
+
+  /**
+   * Calculate directory size in MB
+   */
+  private getDirectorySize(dirPath: string): number {
+    let totalBytes = 0;
+
+    const calculateSize = (currentPath: string) => {
+      const items = fs.readdirSync(currentPath, { withFileTypes: true });
+
+      for (const item of items) {
+        const itemPath = path.join(currentPath, item.name);
+
+        if (item.isDirectory()) {
+          calculateSize(itemPath);
+        } else {
+          try {
+            const stats = fs.statSync(itemPath);
+            totalBytes += stats.size;
+          } catch (error) {
+            // Skip files that can't be read
+          }
+        }
+      }
+    };
+
+    try {
+      calculateSize(dirPath);
+    } catch (error) {
+      return 0;
+    }
+
+    return Math.round(totalBytes / (1024 * 1024)); // Convert to MB
+  }
+
+  /**
+   * Get cache configuration
+   */
+  getConfig() {
+    return {
+      maxSizeGB: this.maxSizeGB,
+      maxRepositories: this.maxRepositories,
+      cacheDir: this.cacheDir,
+    };
+  }
+}
+
+// Global instance initialized with config
+const config = getCacheConfig();
+export const cacheManager = new CacheManager(
+  config.cacheDir,
+  config.maxSizeGB,
+  config.maxRepositories,
+);
diff --git a/src/proxy/processors/push-action/clearBareClone.ts b/src/proxy/processors/push-action/clearBareClone.ts
@@ -1,16 +1,41 @@
 import { Action, Step } from '../../actions';
 import fs from 'node:fs';
 
+const WORK_DIR = './.remote/work';
+
 const exec = async (req: any, action: Action): Promise<Action> => {
   const step = new Step('clearBareClone');
 
-  // Recursively remove the contents of ./.remote and ignore exceptions
-  fs.rm('./.remote', { recursive: true, force: true }, (err) => {
-    if (err) {
-      throw err;
+  // In test environment, clean up EVERYTHING to prevent memory leaks
+  if (process.env.NODE_ENV === 'test') {
+    // TEST: Full cleanup (bare cache + all working copies)
+    try {
+      if (fs.existsSync('./.remote')) {
+        fs.rmSync('./.remote', { recursive: true, force: true });
+        step.log('Test environment: Full .remote directory cleaned');
+      } else {
+        step.log('Test environment: .remote directory already clean');
+      }
+    } catch (err) {
+      step.log(`Warning: Could not clean .remote directory: ${err}`);
+    }
+  } else {
+    // PRODUCTION: Delete ONLY this push's working copy
+    const workCopy = `${WORK_DIR}/${action.id}`;
+
+    if (fs.existsSync(workCopy)) {
+      try {
+        fs.rmSync(workCopy, { recursive: true, force: true });
+        step.log(`Cleaned working copy for push ${action.id}`);
+      } catch (err) {
+        step.log(`Warning: Could not clean working copy ${workCopy}: ${err}`);
+      }
+    } else {
+      step.log(`Working copy ${workCopy} not found (may have been already cleaned)`);
     }
-    console.log(`.remote is deleted!`);
-  });
+
+    step.log('Bare cache preserved for reuse');
+  }
 
   action.addStep(step);
   return action;
diff --git a/src/proxy/processors/push-action/git-operations.ts b/src/proxy/processors/push-action/git-operations.ts
@@ -0,0 +1,122 @@
+import { Step } from '../../actions';
+import { cacheManager } from './cache-manager';
+
+/**
+ * Git Operations for Hybrid Cache
+ */
+
+/**
+ * Execute a git command with credentials sanitization
+ */
+async function execGitCommand(
+  command: string,
+  step: Step,
+  maxBuffer: number = 50 * 1024 * 1024,
+): Promise<{ stdout: string; stderr: string }> {
+  const { exec } = await import('child_process');
+  const { promisify } = await import('util');
+  const execAsync = promisify(exec);
+
+  const { stdout, stderr } = await execAsync(command, { maxBuffer });
+
+  if (stdout) step.log(stdout.trim());
+  if (stderr) step.log(stderr.trim());
+
+  return { stdout, stderr };
+}
+
+/**
+ * Build URL with embedded credentials
+ */
+function buildUrlWithCredentials(url: string, username: string, password: string): string {
+  return url.replace('://', `://${encodeURIComponent(username)}:${encodeURIComponent(password)}@`);
+}
+
+/**
+ * Remove credentials from bare repository config
+ */
+async function sanitizeRepositoryConfig(bareRepo: string, cleanUrl: string): Promise<void> {
+  const { exec } = await import('child_process');
+  const { promisify } = await import('util');
+  const execAsync = promisify(exec);
+
+  // Remove any URL with credentials
+  await execAsync(`cd "${bareRepo}" && git config --unset remote.origin.url 2>/dev/null || true`);
+  // Set clean URL without credentials
+  await execAsync(`cd "${bareRepo}" && git config remote.origin.url "${cleanUrl}"`);
+}
+
+/**
+ * Clone working copy from bare repository using native git
+ */
+export async function cloneWorkingCopy(
+  bareRepo: string,
+  workCopyPath: string,
+  step: Step,
+): Promise<void> {
+  try {
+    await execGitCommand(`git clone "${bareRepo}" "${workCopyPath}"`, step);
+    step.log(`Working copy created at ${workCopyPath}`);
+  } catch (error: any) {
+    step.log(`Failed to create working copy: ${error.message}`);
+    throw error;
+  }
+}
+
+/**
+ * Fetch updates in bare repository using native git command
+ */
+export async function fetchBareRepository(
+  bareRepo: string,
+  url: string,
+  username: string,
+  password: string,
+  step: Step,
+): Promise<void> {
+  const urlWithCreds = buildUrlWithCredentials(url, username, password);
+
+  try {
+    // Fetch all branches with depth=1
+    await execGitCommand(
+      `cd "${bareRepo}" && git fetch --depth=1 "${urlWithCreds}" "+refs/heads/*:refs/heads/*"`,
+      step,
+    );
+
+    // SECURITY: Remove credentials from config
+    await sanitizeRepositoryConfig(bareRepo, url);
+
+    step.log(`Bare repository updated (credentials removed)`);
+  } catch (error: any) {
+    step.log(`Failed to fetch bare repository: ${error.message}`);
+    throw error;
+  }
+}
+
+/**
+ * Clone bare repository using native git command
+ */
+export async function cloneBareRepository(
+  bareRepo: string,
+  url: string,
+  username: string,
+  password: string,
+  step: Step,
+): Promise<void> {
+  const urlWithCreds = buildUrlWithCredentials(url, username, password);
+
+  try {
+    await execGitCommand(`git clone --bare --depth=1 "${urlWithCreds}" "${bareRepo}"`, step);
+
+    // SECURITY: Remove credentials from config immediately after clone
+    await sanitizeRepositoryConfig(bareRepo, url);
+
+    step.log(`Bare repository created at ${bareRepo} (credentials sanitized)`);
+
+    // Update access time for LRU after successful clone
+    const repoName = bareRepo.split('/').pop() || '';
+    cacheManager.touchRepository(repoName);
+  } catch (error: any) {
+    step.log(`Failed to clone bare repository: ${error.message}`);
+    throw error;
+  }
+}
diff --git a/src/proxy/processors/push-action/metrics.ts b/src/proxy/processors/push-action/metrics.ts
@@ -0,0 +1,37 @@
+import { Step } from '../../actions';
+
+/**
+ * Performance Timer
+ *
+ * Logs basic timing info for operations
+ */
+export class PerformanceTimer {
+  private step: Step;
+  private startTime: number = 0;
+  private operation: string = '';
+
+  constructor(step: Step) {
+    this.step = step;
+  }
+
+  start(operation: string): void {
+    this.operation = operation;
+    this.startTime = Date.now();
+    this.step.log(`🚀 ${operation} started`);
+  }
+
+  mark(message: string): void {
+    if (this.startTime > 0) {
+      const elapsed = Date.now() - this.startTime;
+      this.step.log(`⚡ ${message}: ${elapsed}ms`);
+    }
+  }
+
+  end(): void {
+    if (this.startTime > 0) {
+      const totalTime = Date.now() - this.startTime;
+      this.step.log(`✅ ${this.operation} completed: ${totalTime}ms`);
+      this.startTime = 0;
+    }
+  }
+}
diff --git a/src/proxy/processors/push-action/pullRemote.ts b/src/proxy/processors/push-action/pullRemote.ts
