Initial eval setup

Author: olesho

Makefile

@@ -1,7 +1,7 @@
 # Makefile for kernel-browser local development
 # Using kernel-images native build system
 
-.PHONY: help build run stop logs clean dev status shell test
+.PHONY: help build rebuild run stop logs clean dev status shell test
 
 # Default target
 help: ## Show this help message
@@ -57,11 +57,24 @@ rebuild-devtools-full: ## Force complete rebuild from scratch (slow, rarely need
 	docker build -f Dockerfile.devtools --no-cache --target devtools-server -t browser-operator-devtools:latest .
 	@echo "✅ DevTools completely rebuilt"
 
-build: init build-devtools ## Build extended image with DevTools frontend
+build: init ## Build extended image with DevTools frontend (smart: only builds DevTools if needed)
 	@echo "🔨 Building extended kernel-browser with DevTools frontend..."
+	@if ! docker images | grep -q "browser-operator-devtools.*latest"; then \
+		echo "📦 DevTools image not found, building it first..."; \
+		echo "   This is a one-time operation and will take ~30 minutes..."; \
+		$(MAKE) --no-print-directory build-devtools; \
+	else \
+		echo "✅ Using existing DevTools image"; \
+	fi
 	docker build -f Dockerfile.local -t kernel-browser:extended .
 	@echo "✅ Extended build complete"
 
+rebuild: init ## Force complete rebuild (including DevTools)
+	@echo "🔄 Force rebuilding everything from scratch..."
+	$(MAKE) --no-print-directory build-devtools
+	docker build -f Dockerfile.local -t kernel-browser:extended .
+	@echo "✅ Complete rebuild finished"
+
 run: ## Run extended container with DevTools (interactive)
 	@echo "🚀 Starting extended kernel-browser with DevTools..."
 	@if [ -n "$(URLS)" ]; then echo "📄 Opening URLs: $(URLS)"; fi

build-local.sh

@@ -362,12 +362,18 @@ class APIServer {
       // Handle nested model configuration directly
       const nestedModelConfig = this.processNestedModelConfig(requestBody);
 
+      // Extract optional URL and wait timeout
+      const targetUrl = requestBody.url || 'about:blank';
+      const waitTimeout = requestBody.wait_timeout || 5000;
+
       const redact = (mk) => ({
         ...mk,
         api_key: mk?.api_key ? `${String(mk.api_key).slice(0, 4)}...` : undefined
       });
       logger.info('Processing responses request:', {
         input: requestBody.input,
+        url: targetUrl,
+        wait_timeout: targetUrl !== 'about:blank' ? waitTimeout : 0,
         modelConfig: {
           main_model: redact(nestedModelConfig.main_model),
           mini_model: redact(nestedModelConfig.mini_model),
@@ -378,10 +384,10 @@ class APIServer {
       // Find a client with existing tabs (not the dummy client)
       const baseClientId = this.findClientWithTabs();
 
-      // Open a new tab for this request
-      logger.info('Opening new tab for responses request', { baseClientId });
+      // Open a new tab for this request at the specified URL
+      logger.info('Opening new tab for responses request', { baseClientId, url: targetUrl });
       const tabResult = await this.evaluationServer.openTab(baseClientId, {
-        url: 'about:blank',
+        url: targetUrl,
         background: false
       });
 
@@ -393,6 +399,12 @@ class APIServer {
       // Wait for the new tab's DevTools to connect
       const tabClient = await this.waitForClientConnection(tabResult.compositeClientId);
 
+      // Wait for page to load if a custom URL was provided
+      if (targetUrl !== 'about:blank') {
+        logger.info('Waiting for page to load', { waitTimeout });
+        await new Promise(resolve => setTimeout(resolve, waitTimeout));
+      }
+
       // Create a dynamic evaluation for this request
       const evaluation = this.createDynamicEvaluationNested(requestBody.input, nestedModelConfig);
 
@@ -484,6 +496,7 @@ class APIServer {
   findClientWithTabs() {
     const clients = this.evaluationServer.getClientManager().getAllClients();
 
+    // First, try to find a client with existing tabs
     for (const client of clients) {
       const tabs = this.evaluationServer.getClientManager().getClientTabs(client.id);
       if (tabs.length > 0) {
@@ -492,7 +505,13 @@ class APIServer {
       }
     }
 
-    throw new Error('No client with existing tabs found. Please ensure at least one DevTools client with a tab is connected.');
+    // If no client with tabs, use the first available client (even with 0 tabs)
+    if (clients.length > 0) {
+      logger.info('No clients with tabs found, using first available client', { clientId: clients[0].id });
+      return clients[0].id;
+    }
+
+    throw new Error('No clients found. Please ensure at least one DevTools client is registered.');
   }
 
   /**
@@ -540,7 +559,7 @@ class APIServer {
       description: 'Dynamic evaluation created from API request',
       enabled: true,
       tool: 'chat',
-      timeout: 1500000, // 25 minutes
+      timeout: 7200000, // 2 hours (increased for slow custom API)
       input: {
         message: input
       },

eval-server/nodejs/src/api-server.js

@@ -45,7 +45,7 @@ export const CONFIG = {
   },
 
   rpc: {
-    timeout: parseInt(process.env.RPC_TIMEOUT) || 1500000, // 25 minutes default
+    timeout: parseInt(process.env.RPC_TIMEOUT) || 7200000, // 2 hours default (increased for slow custom API)
     maxConcurrentEvaluations: parseInt(process.env.MAX_CONCURRENT_EVALUATIONS) || 10
   },
 

eval-server/nodejs/src/config.js

@@ -0,0 +1,15 @@
+# Evaluation Framework Environment Variables
+# Copy this file to .env and fill in your actual API keys
+
+# Required: OpenAI API key for LLM judge and main model
+OPENAI_API_KEY=sk-your-openai-api-key-here
+
+# Optional: Groq API key (if using Groq models)
+GROQ_API_KEY=gsk-your-groq-api-key-here
+
+# Optional: OpenRouter API key (if using OpenRouter)
+OPENROUTER_API_KEY=your-openrouter-api-key-here
+
+# Optional: LiteLLM configuration (if using LiteLLM)
+LITELLM_API_KEY=your-litellm-api-key-here
+LITELLM_ENDPOINT=http://localhost:8000

evals/.env.example

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# uv
+.uv/
+uv.lock
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Reports
+reports/*.csv
+
+# Environment variables
+.env
+.env.local
+
+# OS
+.DS_Store
+Thumbs.db

evals/.gitignore

@@ -0,0 +1 @@
+3.9