Skip to content

Commit 6846950

Browse files
jahoomajahooma
committed
Add best-of-n thinker subagent
1 parent b7a590e commit 6846950

File tree

6 files changed

+435
-3
lines changed

6 files changed

+435
-3
lines changed

‎.agents/base2/base2.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,10 @@ export function createBase2(
6969
'researcher-web',
7070
'researcher-docs',
7171
'commander',
72-
isGpt5 && 'best-of-n-editor-gpt-5',
7372
isDefault && 'best-of-n-editor',
73+
isGpt5 && 'best-of-n-editor-gpt-5',
74+
isDefault && 'thinker-best-of-n',
75+
isGpt5 && 'thinker-best-of-n-gpt-5',
7476
'context-pruner',
7577
),
7678

‎.agents/thinker/thinker-best-of-n-gpt-5.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { createThinkerBestOfN } from './thinker-best-of-n'
2+
3+
const definition = {
4+
...createThinkerBestOfN('gpt-5'),
5+
id: 'thinker-best-of-n-gpt-5',
6+
}
7+
export default definition

‎.agents/thinker/thinker-best-of-n.ts‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
import { publisher } from '../constants'
2+
3+
import type { SecretAgentDefinition } from '../types/secret-agent-definition'
4+
import type { AgentStepContext, ToolCall } from '../types/agent-definition'
5+
6+
export function createThinkerBestOfN(
7+
model: 'sonnet' | 'gpt-5',
8+
): Omit<SecretAgentDefinition, 'id'> {
9+
const isGpt5 = model === 'gpt-5'
10+
11+
return {
12+
publisher,
13+
model: isGpt5 ? 'openai/gpt-5' : 'anthropic/claude-sonnet-4.5',
14+
displayName: isGpt5 ? 'Best-of-N GPT-5 Thinker' : 'Best-of-N Thinker',
15+
spawnerPrompt:
16+
'Generates deep thinking by orchestrating multiple thinker agents, selects the best thinking output. Use this to help solve a hard problem. You must first gather all the relevant context *BEFORE* spawning this agent, as it can only think.',
17+
18+
includeMessageHistory: true,
19+
inheritParentSystemPrompt: true,
20+
21+
toolNames: ['spawn_agents', 'set_messages', 'set_output'],
22+
spawnableAgents: isGpt5
23+
? ['thinker-gpt-5', 'thinker-selector-gpt-5']
24+
: ['thinker', 'thinker-selector'],
25+
26+
inputSchema: {
27+
prompt: {
28+
type: 'string',
29+
description: 'The problem you are trying to solve',
30+
},
31+
params: {
32+
type: 'object',
33+
properties: {
34+
n: {
35+
type: 'number',
36+
description:
37+
'Number of parallel thinker agents to spawn. Defaults to 5. Use fewer for simple questions and max of 10 for complex questions.',
38+
},
39+
},
40+
},
41+
},
42+
outputMode: 'structured_output',
43+
44+
handleSteps: isGpt5 ? handleStepsGpt5 : handleStepsSonnet,
45+
}
46+
}
47+
48+
function* handleStepsSonnet({
49+
agentState,
50+
prompt,
51+
params,
52+
}: AgentStepContext): ReturnType<
53+
NonNullable<SecretAgentDefinition['handleSteps']>
54+
> {
55+
const thinkerAgent = 'thinker'
56+
const selectorAgent = 'thinker-selector'
57+
const n = Math.min(10, Math.max(1, (params?.n as number | undefined) ?? 5))
58+
59+
// Remove userInstruction message for this agent.
60+
const messages = agentState.messageHistory.concat()
61+
messages.pop()
62+
yield {
63+
toolName: 'set_messages',
64+
input: {
65+
messages,
66+
},
67+
includeToolCall: false,
68+
} satisfies ToolCall<'set_messages'>
69+
70+
const { toolResult: thinkersResult1 } = yield {
71+
toolName: 'spawn_agents',
72+
input: {
73+
agents: Array.from({ length: n }, () => ({
74+
agent_type: thinkerAgent,
75+
prompt,
76+
})),
77+
},
78+
includeToolCall: false,
79+
} satisfies ToolCall<'spawn_agents'>
80+
81+
const thinkersResult = extractSpawnResults<string>(thinkersResult1)
82+
83+
// Extract all the thinking outputs
84+
const letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
85+
const thoughts = thinkersResult.map((content, index) => ({
86+
id: letters[index],
87+
content,
88+
}))
89+
90+
// Spawn selector with thoughts as params
91+
const { toolResult: selectorResult } = yield {
92+
toolName: 'spawn_agents',
93+
input: {
94+
agents: [
95+
{
96+
agent_type: selectorAgent,
97+
params: { thoughts },
98+
},
99+
],
100+
},
101+
includeToolCall: false,
102+
} satisfies ToolCall<'spawn_agents'>
103+
104+
const selectorOutput = extractSpawnResults<{
105+
thoughtId: string
106+
}>(selectorResult)[0]
107+
108+
if ('errorMessage' in selectorOutput) {
109+
yield {
110+
toolName: 'set_output',
111+
input: { error: selectorOutput.errorMessage },
112+
} satisfies ToolCall<'set_output'>
113+
return
114+
}
115+
const { thoughtId } = selectorOutput
116+
const chosenThought = thoughts.find((thought) => thought.id === thoughtId)
117+
if (!chosenThought) {
118+
yield {
119+
toolName: 'set_output',
120+
input: { error: 'Failed to find chosen thinking output.' },
121+
} satisfies ToolCall<'set_output'>
122+
return
123+
}
124+
125+
// Set output with the chosen thinking
126+
yield {
127+
toolName: 'set_output',
128+
input: {
129+
response: chosenThought.content,
130+
},
131+
includeToolCall: false,
132+
} satisfies ToolCall<'set_output'>
133+
134+
function extractSpawnResults<T>(
135+
results: any[] | undefined,
136+
): (T | { errorMessage: string })[] {
137+
if (!results) return []
138+
const spawnedResults = results
139+
.filter((result) => result.type === 'json')
140+
.map((result) => result.value)
141+
.flat() as {
142+
agentType: string
143+
value: { value?: T; errorMessage?: string }
144+
}[]
145+
return spawnedResults.map(
146+
(result) =>
147+
result.value.value ??
148+
({
149+
errorMessage:
150+
result.value.errorMessage ?? 'Error extracting spawn results',
151+
} as { errorMessage: string }),
152+
)
153+
}
154+
}
155+
156+
function* handleStepsGpt5({
157+
agentState,
158+
prompt,
159+
params,
160+
}: AgentStepContext): ReturnType<
161+
NonNullable<SecretAgentDefinition['handleSteps']>
162+
> {
163+
const thinkerAgent = 'thinker-gpt-5'
164+
const selectorAgent = 'thinker-selector-gpt-5'
165+
const n = Math.min(10, Math.max(1, (params?.n as number | undefined) ?? 5))
166+
167+
// Remove userInstruction message for this agent.
168+
const messages = agentState.messageHistory.concat()
169+
messages.pop()
170+
yield {
171+
toolName: 'set_messages',
172+
input: {
173+
messages,
174+
},
175+
includeToolCall: false,
176+
} satisfies ToolCall<'set_messages'>
177+
178+
const { toolResult: thinkersResult1 } = yield {
179+
toolName: 'spawn_agents',
180+
input: {
181+
agents: Array.from({ length: n }, () => ({
182+
agent_type: thinkerAgent,
183+
prompt,
184+
})),
185+
},
186+
includeToolCall: false,
187+
} satisfies ToolCall<'spawn_agents'>
188+
189+
const thinkersResult = extractSpawnResults<string>(thinkersResult1)
190+
191+
// Extract all the thinking outputs
192+
const letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
193+
const thoughts = thinkersResult.map((content, index) => ({
194+
id: letters[index],
195+
content,
196+
}))
197+
198+
// Spawn selector with thoughts as params
199+
const { toolResult: selectorResult } = yield {
200+
toolName: 'spawn_agents',
201+
input: {
202+
agents: [
203+
{
204+
agent_type: selectorAgent,
205+
params: { thoughts },
206+
},
207+
],
208+
},
209+
includeToolCall: false,
210+
} satisfies ToolCall<'spawn_agents'>
211+
212+
const selectorOutput = extractSpawnResults<{
213+
thoughtId: string
214+
}>(selectorResult)[0]
215+
216+
if ('errorMessage' in selectorOutput) {
217+
yield {
218+
toolName: 'set_output',
219+
input: { error: selectorOutput.errorMessage },
220+
} satisfies ToolCall<'set_output'>
221+
return
222+
}
223+
const { thoughtId } = selectorOutput
224+
const chosenThought = thoughts.find((thought) => thought.id === thoughtId)
225+
if (!chosenThought) {
226+
yield {
227+
toolName: 'set_output',
228+
input: { error: 'Failed to find chosen thinking output.' },
229+
} satisfies ToolCall<'set_output'>
230+
return
231+
}
232+
233+
// Set output with the chosen thinking
234+
yield {
235+
toolName: 'set_output',
236+
input: {
237+
response: chosenThought.content,
238+
},
239+
includeToolCall: false,
240+
} satisfies ToolCall<'set_output'>
241+
242+
function extractSpawnResults<T>(
243+
results: any[] | undefined,
244+
): (T | { errorMessage: string })[] {
245+
if (!results) return []
246+
const spawnedResults = results
247+
.filter((result) => result.type === 'json')
248+
.map((result) => result.value)
249+
.flat() as {
250+
agentType: string
251+
value: { value?: T; errorMessage?: string }
252+
}[]
253+
return spawnedResults.map(
254+
(result) =>
255+
result.value.value ??
256+
({
257+
errorMessage:
258+
result.value.errorMessage ?? 'Error extracting spawn results',
259+
} as { errorMessage: string }),
260+
)
261+
}
262+
}
263+
264+
const definition: SecretAgentDefinition = {
265+
...createThinkerBestOfN('sonnet'),
266+
id: 'thinker-best-of-n',
267+
}
268+
269+
export default definition

‎.agents/thinker/thinker-selector-gpt-5.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import { type SecretAgentDefinition } from '../types/secret-agent-definition'
2+
import { publisher } from '../constants'
3+
4+
const definition: SecretAgentDefinition = {
5+
id: 'thinker-selector-gpt-5',
6+
publisher,
7+
model: 'openai/gpt-5',
8+
displayName: 'Thinker Output Selector GPT-5',
9+
spawnerPrompt: 'Analyzes multiple thinking outputs and selects the best one',
10+
11+
includeMessageHistory: true,
12+
inheritParentSystemPrompt: true,
13+
14+
toolNames: ['set_output'],
15+
spawnableAgents: [],
16+
17+
inputSchema: {
18+
params: {
19+
type: 'object',
20+
properties: {
21+
thoughts: {
22+
type: 'array',
23+
items: {
24+
type: 'object',
25+
properties: {
26+
id: { type: 'string' },
27+
content: { type: 'string' },
28+
},
29+
required: ['id', 'content'],
30+
},
31+
},
32+
},
33+
required: ['thoughts'],
34+
},
35+
},
36+
outputMode: 'structured_output',
37+
outputSchema: {
38+
type: 'object',
39+
properties: {
40+
thoughtId: {
41+
type: 'string',
42+
description: 'The id of the chosen thinking output',
43+
},
44+
},
45+
required: ['thoughtId'],
46+
},
47+
48+
instructionsPrompt: `As part of the best-of-n workflow for thinking agents, you are the thinking selector agent.
49+
50+
## Task Instructions
51+
52+
You have been provided with multiple thinking outputs via params.
53+
54+
The thoughts are available in the params.thoughts array, where each has:
55+
- id: A unique identifier for the thinking output
56+
- content: The full thinking text
57+
58+
Your task is to analyze each thinking output carefully, compare them against the original user question, and select the best thinking.
59+
Evaluate each based on (in order of importance):
60+
- Depth and thoroughness in addressing the user's question.
61+
- Correctness and accuracy of insights.
62+
- Clarity and organization of thoughts.
63+
- Practical actionability of recommendations.
64+
- Consideration of edge cases and alternatives.
65+
66+
## User Request
67+
68+
Try to select the thinking output that best answers the user's problem.
69+
70+
## Response Format
71+
72+
If the best one is obvious or the outputs are very similar, you may not need to think very much (a few words suffice) or you may not need to use think tags at all, just pick the best one and output it. You have a dual goal of picking the best thinking and being fast (using as few words as possible).
73+
74+
Then, do not write any other explanations AT ALL. You should directly output a single tool call to set_output with the selected thoughtId.`,
75+
}
76+
77+
export default definition

0 commit comments

Comments
 (0)