Debug error option in agent runner

Author: jahooma

evals/buffbench/agent-runner.ts

@@ -1,3 +1,5 @@
+import fs from 'fs'
+import path from 'path'
 import { execSync } from 'child_process'
 
 import { withTimeout } from '@codebuff/common/util/promise'
@@ -9,6 +11,8 @@ import type { EvalCommitV2 } from './types'
 
 export type AgentStep = PrintModeEvent
 
+const DEBUG_ERROR = false
+
 export async function runAgentOnCommit({
   client,
   agentId,
@@ -66,6 +70,22 @@ export async function runAgentOnCommit({
                   `[${commit.id}:${agentId}] Error event:`,
                   event.message,
                 )
+                if (DEBUG_ERROR) {
+                  fs.writeFileSync(
+                    path.join(
+                      __dirname,
+                      `${commit.id}-${agentId}-error-${Math.random().toString(36).substring(2, 6)}.json`,
+                    ),
+                    JSON.stringify(
+                      {
+                        error: event.message,
+                        trace: trace,
+                      },
+                      null,
+                      2,
+                    ),
+                  )
+                }
               }
               trace.push(event)
             },

evals/buffbench/judge.ts

@@ -182,8 +182,8 @@ ${error ? `\n## Error Encountered\n${error}` : ''}`
         }
       },
     }),
-    10 * 60 * 1000,
-    'Judge agent timed out after 10 minutes',
+    20 * 60 * 1000,
+    'Judge agent timed out after 20 minutes',
   )
 
   if (judgeResult.output.type !== 'structuredOutput') {

evals/buffbench/trace-analyzer.ts

@@ -23,7 +23,7 @@ function truncateTrace(trace: AgentStep[]): AgentStep[] {
     // Handle tool_result events
     if (step.type === 'tool_result') {
       const output = Array.isArray(step.output) ? step.output : [step.output]
-      
+
       // Truncate read_files results
       if (step.toolName === 'read_files') {
         const truncatedOutput = output.map((item: any) => {
@@ -96,7 +96,7 @@ function truncateTrace(trace: AgentStep[]): AgentStep[] {
         }
       }
     }
-    
+
     return step
   })
 }
@@ -278,8 +278,8 @@ Focus on the HOW, not the WHAT: We want to understand and improve how agents wor
           }
         },
       }),
-      10 * 60 * 1000,
-      'Trace analyzer agent timed out after 10 minutes',
+      20 * 60 * 1000,
+      'Trace analyzer agent timed out after 20 minutes',
     )
 
     const { output } = analyzerResult