implement tools synchronization with cache optimization

.gitignore

@@ -39,3 +39,7 @@ yarn-error.log*
 
 .cursor/
 .vscode/
+
+# Docker Files
+docker-compose.override.yml
+compose.override.yml

Dockerfile.dev

@@ -35,7 +35,7 @@ ENV CHOKIDAR_USEPOLLING=true
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY turbo.json ./
 
-# Copy package.json files from all workspaces
+# Copy package.json files from all workspaces (needed for dependency installation)
 COPY apps/frontend/package.json ./apps/frontend/
 COPY apps/backend/package.json ./apps/backend/
 COPY packages/eslint-config/package.json ./packages/eslint-config/
@@ -46,8 +46,10 @@ COPY packages/zod-types/package.json ./packages/zod-types/
 # Install all dependencies (including dev dependencies)
 RUN pnpm install
 
-# Create necessary directories for volume mounts
-RUN mkdir -p apps/frontend apps/backend packages/trpc packages/zod-types packages/eslint-config packages/typescript-config
+# Copy source code directories (will be overridden by volume mounts, but ensures structure exists)
+# This ensures the directory structure is present even if volumes don't mount correctly
+COPY apps ./apps
+COPY packages ./packages
 
 # Expose ports
 EXPOSE 12008 12009

apps/backend/package.json

@@ -13,6 +13,9 @@
     "clean": "rm -rf dist",
     "lint": "eslint . --max-warnings 0",
     "lint:fix": "eslint . --fix",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
     "db:generate": "dotenv -e ../../.env -- drizzle-kit generate",
     "db:generate:dev": "dotenv -e ../../.env.local -- drizzle-kit generate",
     "db:migrate": "dotenv -e ../../.env -- drizzle-kit migrate",
@@ -47,12 +50,14 @@
     "@types/node": "^20.0.0",
     "@types/pg": "^8.15.4",
     "@types/shell-quote": "^1.7.5",
+    "@vitest/coverage-v8": "^4.0.0",
     "dotenv-cli": "^8.0.0",
     "drizzle-kit": "^0.31.1",
     "eslint": "^9.28.0",
     "tsup": "^8.5.0",
     "tsx": "^4.20.3",
-    "typescript": "^5.0.0"
+    "typescript": "^5.0.0",
+    "vitest": "^4.0.0"
   },
   "keywords": [],
   "author": "",

apps/backend/src/db/repositories/tools.repo.ts

@@ -3,7 +3,7 @@ import {
   ToolCreateInput,
   ToolUpsertInput,
 } from "@repo/zod-types";
-import { eq, sql } from "drizzle-orm";
+import { and, eq, notInArray, sql } from "drizzle-orm";
 
 import { db } from "../index";
 import { toolsTable } from "../schema";
@@ -40,7 +40,7 @@ export class ToolsRepository {
     }));
 
     // Batch insert all tools with upsert
-    return await db
+    const result = await db
       .insert(toolsTable)
       .values(toolsToInsert)
       .onConflictDoUpdate({
@@ -52,6 +52,8 @@ export class ToolsRepository {
         },
       })
       .returning();
+
+    return result;
   }
 
   async findByUuid(uuid: string): Promise<DatabaseTool | undefined> {
@@ -72,6 +74,62 @@ export class ToolsRepository {
 
     return deletedTool;
   }
+
+  /**
+   * Delete tools that are no longer present in the current tool list
+   * @param mcpServerUuid - UUID of the MCP server
+   * @param currentToolNames - Array of tool names that currently exist in the MCP server
+   * @returns Array of deleted tools
+   */
+  async deleteObsoleteTools(
+    mcpServerUuid: string,
+    currentToolNames: string[],
+  ): Promise<DatabaseTool[]> {
+    if (currentToolNames.length === 0) {
+      // If no tools are provided, delete all tools for this server
+      return await db
+        .delete(toolsTable)
+        .where(eq(toolsTable.mcp_server_uuid, mcpServerUuid))
+        .returning();
+    }
+
+    // Delete tools that are in DB but not in current tool list
+    return await db
+      .delete(toolsTable)
+      .where(
+        and(
+          eq(toolsTable.mcp_server_uuid, mcpServerUuid),
+          notInArray(toolsTable.name, currentToolNames),
+        ),
+      )
+      .returning();
+  }
+
+  /**
+   * Sync tools for a server: upsert current tools and delete obsolete ones
+   * @param input - Tool upsert input containing tools and server UUID
+   * @returns Object with upserted and deleted tools
+   */
+  async syncTools(input: ToolUpsertInput): Promise<{
+    upserted: DatabaseTool[];
+    deleted: DatabaseTool[];
+  }> {
+    const currentToolNames = input.tools.map((tool) => tool.name);
+
+    // First, delete obsolete tools
+    const deleted = await this.deleteObsoleteTools(
+      input.mcpServerUuid,
+      currentToolNames,
+    );
+
+    // Then, upsert current tools
+    let upserted: DatabaseTool[] = [];
+    if (input.tools.length > 0) {
+      upserted = await this.bulkUpsert(input);
+    }
+
+    return { upserted, deleted };
+  }
 }
 
 export const toolsRepository = new ToolsRepository();

apps/backend/src/lib/metamcp/metamcp-proxy.ts

@@ -26,6 +26,7 @@ import { configService } from "../config.service";
 import { ConnectedClient } from "./client";
 import { getMcpServers } from "./fetch-metamcp";
 import { mcpServerPool } from "./mcp-server-pool";
+import { toolsSyncCache } from "./tools-sync-cache";
 import {
   createFilterCallToolMiddleware,
   createFilterListToolsMiddleware,
@@ -143,6 +144,8 @@ export const createServer = async (
     request,
     context,
   ) => {
+    console.log("[DEBUG-TOOLS] 🔍 tools/list called for namespace:", namespaceUuid);
+    const startTime = performance.now();
     const serverParams = await getMcpServers(
       context.namespaceUuid,
       includeInactiveServers,
@@ -155,10 +158,15 @@ export const createServer = async (
     // We'll filter servers during processing after getting sessions to check actual MCP server names
     const allServerEntries = Object.entries(serverParams);
 
+    console.log(`[DEBUG-TOOLS] 📋 Processing ${allServerEntries.length} servers`);
+
     await Promise.allSettled(
       allServerEntries.map(async ([mcpServerUuid, params]) => {
+        console.log(`[DEBUG-TOOLS] 🔧 Server: ${params.name || mcpServerUuid}`);
+
         // Skip if we've already visited this server to prevent circular references
         if (visitedServers.has(mcpServerUuid)) {
+          console.log(`[DEBUG-TOOLS] ⏭️  Skipping already visited: ${params.name}`);
           return;
         }
         const session = await mcpServerPool.getSession(
@@ -167,7 +175,10 @@ export const createServer = async (
           params,
           namespaceUuid,
         );
-        if (!session) return;
+        if (!session) {
+          console.log(`[DEBUG-TOOLS] ❌ No session for: ${params.name}`);
+          return;
+        }
 
         // Now check for self-referencing using the actual MCP server name
         const serverVersion = session.client.getServerVersion();
@@ -201,6 +212,7 @@ export const createServer = async (
           const allServerTools: Tool[] = [];
           let cursor: string | undefined = undefined;
           let hasMore = true;
+          const toolFetchStart = performance.now();
 
           while (hasMore) {
             const result: z.infer<typeof ListToolsResultSchema> =
@@ -222,30 +234,42 @@ export const createServer = async (
             cursor = result.nextCursor;
             hasMore = !!result.nextCursor;
           }
+
+          console.log(`[DEBUG-TOOLS] ⏱️  Fetched ${allServerTools.length} tools from ${serverName} in ${(performance.now() - toolFetchStart).toFixed(2)}ms`);
 
           // Save original tools to database (before middleware processing)
           // This ensures we only save the actual tool names, not override names
           // Filter out tools that are overrides of existing tools to prevent duplicates
-          if (allServerTools.length > 0) {
-            try {
+          try {
+            // PERFORMANCE OPTIMIZATION: Check hash FIRST to avoid expensive operations
+            const toolNames = allServerTools.map((tool) => tool.name);
+            const hasChanged = toolsSyncCache.hasChanged(mcpServerUuid, toolNames);
+
+            console.log(`[DEBUG-TOOLS] 🔍 Hash check for ${serverName}: ${hasChanged ? 'CHANGED' : 'UNCHANGED'}`);
+
+            if (hasChanged) {
               const toolsToSave = await filterOutOverrideTools(
                 allServerTools,
                 namespaceUuid,
                 serverName,
               );
 
               if (toolsToSave.length > 0) {
-                await toolsImplementations.create({
+                // Update cache
+                toolsSyncCache.update(mcpServerUuid, toolNames);
+
+                // Sync with cleanup
+                await toolsImplementations.sync({
                   tools: toolsToSave,
                   mcpServerUuid: mcpServerUuid,
                 });
               }
-            } catch (dbError) {
-              console.error(
-                `Error saving tools to database for server ${serverName}:`,
-                dbError,
-              );
             }
+          } catch (dbError) {
+            console.error(
+              `Error syncing tools to database for server ${serverName}:`,
+              dbError,
+            );
           }
 
           // Use original tools for client response (middleware will be applied later)
@@ -268,6 +292,9 @@ export const createServer = async (
       }),
     );
 
+    const totalTime = performance.now() - startTime;
+    console.log(`[DEBUG-TOOLS] ✅ tools/list completed in ${totalTime.toFixed(2)}ms, returning ${allTools.length} tools`);
+
     return { tools: allTools };
   };