Skip to content

Commit 8a401b3

Browse files
committed
refactor: do not generate user journeys when they are not used
Arguably these user journeys may be nice to show in the UI of individual apps, but without any testing there, it doesn't make sense to generate them and consume more tokens.
1 parent 365468e commit 8a401b3

File tree

1 file changed

+15
-14
lines changed

1 file changed

+15
-14
lines changed

runner/orchestration/generate.ts

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -322,23 +322,24 @@ async function startEvaluationTask(
322322
break;
323323
}
324324

325-
const userJourneys = await generateUserJourneysForApp(
326-
ratingLlm,
327-
rootPromptDef.name,
328-
defsToExecute[0].prompt,
329-
initialResponse.files,
330-
abortSignal,
331-
);
325+
const userJourneys = config.enableUserJourneyTesting
326+
? await generateUserJourneysForApp(
327+
ratingLlm,
328+
rootPromptDef.name,
329+
defsToExecute[0].prompt,
330+
initialResponse.files,
331+
abortSignal,
332+
)
333+
: undefined;
332334

333335
// TODO: Only execute the serve command on the "final working attempt".
334336
// TODO: Incorporate usage.
335-
const userJourneyAgentTaskInput: BrowserAgentTaskInput | undefined =
336-
config.enableUserJourneyTesting
337-
? {
338-
userJourneys: userJourneys.result,
339-
appPrompt: defsToExecute[0].prompt,
340-
}
341-
: undefined;
337+
const userJourneyAgentTaskInput: BrowserAgentTaskInput | undefined = userJourneys
338+
? {
339+
userJourneys: userJourneys.result,
340+
appPrompt: defsToExecute[0].prompt,
341+
}
342+
: undefined;
342343

343344
const attemptDetails: AttemptDetails[] = []; // Store details for assessment.json
344345

0 commit comments

Comments
 (0)