diff --git a/package.json b/package.json index e44407b..2bd0953 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "web-codegen-scorer", - "version": "0.0.21", + "version": "0.0.23", "scripts": { "build-runner": "tsc", "release-build": "tsx ./scripts/release-build.ts", diff --git a/report-app/src/app/shared/provider-label.ts b/report-app/src/app/shared/provider-label.ts index b3ab5af..2367150 100644 --- a/report-app/src/app/shared/provider-label.ts +++ b/report-app/src/app/shared/provider-label.ts @@ -30,7 +30,8 @@ const exactMatches: Record = { height: 24px; } - :host-context(.dark-mode) :host(.genkit) .logo { + :host-context(.dark-mode) :host(.genkit) .logo, + :host-context(.dark-mode) :host(.next) .logo { filter: invert(1); } `, diff --git a/runner/orchestration/generate-summary.ts b/runner/orchestration/generate-summary.ts index da3b410..c48e87a 100644 --- a/runner/orchestration/generate-summary.ts +++ b/runner/orchestration/generate-summary.ts @@ -44,13 +44,17 @@ export async function prepareSummary( console.log(`✨ Generating AI summary for evaluation run..`); try { const result = await summarizeReportWithAI(genkit, abortSignal, assessments); - inputTokens += result.usage.inputTokens; - outputTokens += result.usage.outputTokens; - totalTokens += result.usage.totalTokens; - aiSummary = result.responseHtml; - console.log(`✅ Generated AI summary.`); + + if (result !== null) { + inputTokens += result.usage.inputTokens; + outputTokens += result.usage.outputTokens; + totalTokens += result.usage.totalTokens; + aiSummary = result.responseHtml; + console.log(`✅ Generated AI summary.`); + } } catch (e) { console.log(`${redX()} Failed to generate AI summary, skipping summary.`); + if ((e as Partial).stack) { console.error((e as Error).stack); } diff --git a/runner/reporting/report-ai-summary.ts b/runner/reporting/report-ai-summary.ts index be4adf7..e84fe62 100644 --- a/runner/reporting/report-ai-summary.ts +++ b/runner/reporting/report-ai-summary.ts @@ -1,10 +1,5 @@ import {GenkitRunner} from '../codegen/genkit/genkit-runner.js'; -import { - AssessmentResult, - ReportContextFilter, - RatingContextFilter, - AiChatContextFilters, -} from '../shared-interfaces.js'; +import {AssessmentResult, ReportContextFilter, RatingContextFilter} from '../shared-interfaces.js'; import {chatWithReportAI} from './report-ai-chat.js'; export async function summarizeReportWithAI( @@ -12,6 +7,12 @@ export async function summarizeReportWithAI( abortSignal: AbortSignal, assessments: AssessmentResult[], ) { + const model = 'gemini-2.5-flash-lite'; + + if (!llm.getSupportedModels().includes(model)) { + return null; + } + return chatWithReportAI( llm, `Strictly follow the instructions here. @@ -31,7 +32,7 @@ Categorize the failures and provide a brief summary of the report. Keep it short assessments, [], // For AI summaries we use lite model as it's faster and cheaper (+ reduces rate limiting) - 'gemini-2.5-flash-lite', + model, { reportContextFilter: ReportContextFilter.NonPerfectReports, ratingContextFilter: RatingContextFilter.NonPerfectRatings,