From abe03d169ab64ccb662734d5536aceb0d4c75967 Mon Sep 17 00:00:00 2001 From: andresgutgon Date: Fri, 14 Nov 2025 17:25:38 +0100 Subject: [PATCH 1/2] Issues banner Implement a banner in runs to make users aware that they need to annotate logs in order to start using issues so they can see where are the biggest problems on their prompts. --- .../runs/_components/IssuesOverviewCard.tsx | 93 +++++++++++++++++++ .../runs/_components/RunsPage.tsx | 2 + .../[projectId]/issues/overview/route.ts | 41 ++++++++ packages/constants/src/issues/constants.ts | 4 + .../issues/annotatedResultStatsByProject.ts | 22 +++++ .../evaluationResultsV2Repository.ts | 26 +++++- .../core/src/repositories/issuesRepository.ts | 15 ++- 7 files changed, 197 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx create mode 100644 apps/web/src/app/api/projects/[projectId]/issues/overview/route.ts create mode 100644 packages/core/src/data-access/issues/annotatedResultStatsByProject.ts diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx new file mode 100644 index 0000000000..fe8a5b20a2 --- /dev/null +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx @@ -0,0 +1,93 @@ +'use client' + +import { Button } from '@latitude-data/web-ui/atoms/Button' +import { Text } from '@latitude-data/web-ui/atoms/Text' +import { Icon } from '@latitude-data/web-ui/atoms/Icons' +import { useCurrentProject } from '$/app/providers/ProjectProvider' +import { useCurrentCommit } from '$/app/providers/CommitProvider' +import { ROUTES } from '$/services/routes' +import { useRouter } from 'next/navigation' +import useSWR from 'swr' +import { IssuesOverviewResponse } from '$/app/api/projects/[projectId]/issues/overview/route' + +const fetcher = (url: string) => fetch(url).then((r) => r.json()) + +export function IssuesOverviewCard() { + const { project } = useCurrentProject() + const { commit } = useCurrentCommit() + const router = useRouter() + + const { data, isLoading } = useSWR( + `/api/projects/${project.id}/issues/overview`, + fetcher, + ) + + if (isLoading || !data) return null + if (data.totalAnnotations === 0) return null + + const progress = data.totalAnnotations > 0 + ? (data.issuesWithAnnotations / data.totalAnnotations) * 100 + : 0 + + const handleReviewIssues = () => { + router.push( + ROUTES.projects + .detail({ id: project.id }) + .commits.detail({ uuid: commit.uuid }) + .issues.root, + ) + } + + return ( +
+
+
+ Issues overview is now available! + + Keep annotating to discover more issues. + +
+ +
+ +
+
+ {/* Background bar */} +
+ {/* Progress bar */} +
+
+
+ + {/* Lightbulb icon */} +
+
+ +
+ +
+ +
+
+ +
+
+ Insight discovery ยท + {data.issuesWithAnnotations} +
+ {data.totalAnnotations} +
+
+ ) +} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx index 6dd9da0189..0701587883 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx @@ -29,6 +29,7 @@ import { AppLocalStorage, useLocalStorage, } from '@latitude-data/web-ui/hooks/useLocalStorage' +import { IssuesOverviewCard } from './IssuesOverviewCard' function sumCounts( counts: Record | undefined, @@ -284,6 +285,7 @@ export function RunsPage({ /> ) : (
+
No run selected
diff --git a/apps/web/src/app/api/projects/[projectId]/issues/overview/route.ts b/apps/web/src/app/api/projects/[projectId]/issues/overview/route.ts new file mode 100644 index 0000000000..da64b0a32b --- /dev/null +++ b/apps/web/src/app/api/projects/[projectId]/issues/overview/route.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' +import { authHandler } from '$/middlewares/authHandler' +import { errorHandler } from '$/middlewares/errorHandler' +import { ProjectsRepository } from '@latitude-data/core/repositories' +import { Workspace } from '@latitude-data/core/schema/models/types/Workspace' +import { NextRequest, NextResponse } from 'next/server' +import { annotatedResultStatsByProject } from '@latitude-data/core/data-access/issues/annotatedResultStatsByProject' + +const paramsSchema = z.object({ + projectId: z.coerce.number(), +}) + +export type IssuesOverviewResponse = Awaited< + ReturnType +> + +export const GET = errorHandler( + authHandler( + async ( + _request: NextRequest, + { + params, + workspace, + }: { + params: { + projectId: string + } + workspace: Workspace + }, + ) => { + const { projectId } = paramsSchema.parse({ + projectId: params.projectId, + }) + const projectsRepo = new ProjectsRepository(workspace.id) + const project = await projectsRepo.find(projectId).then((r) => r.unwrap()) + const stats = await annotatedResultStatsByProject({ project }) + + return NextResponse.json(stats, { status: 200 }) + }, + ), +) diff --git a/packages/constants/src/issues/constants.ts b/packages/constants/src/issues/constants.ts index 94f1c3a7d3..227e19ee3e 100644 --- a/packages/constants/src/issues/constants.ts +++ b/packages/constants/src/issues/constants.ts @@ -97,3 +97,7 @@ export const ISSUE_JOBS_MAX_ATTEMPTS = 3 export const ISSUE_JOBS_GENERATE_DETAILS_THROTTLE = 6 * 60 * 60 * 1000 // 6 hours export const ISSUE_JOBS_MERGE_COMMON_THROTTLE = 24 * 60 * 60 * 1000 // 1 day export const ISSUE_JOBS_DISCOVER_RESULT_DELAY = 60 * 1000 // 1 minute + +// This is the minimum number of manually annotated evaluation results required +// for automatic issue creation to be enabled for a project. +export const MINIMUM_ANNOTATED_THRESHOLD = 10 diff --git a/packages/core/src/data-access/issues/annotatedResultStatsByProject.ts b/packages/core/src/data-access/issues/annotatedResultStatsByProject.ts new file mode 100644 index 0000000000..26c1cc4a53 --- /dev/null +++ b/packages/core/src/data-access/issues/annotatedResultStatsByProject.ts @@ -0,0 +1,22 @@ +import { + EvaluationResultsV2Repository, + IssuesRepository, +} from '../../repositories' +import { Project } from '../../schema/models/types/Project' + +export async function annotatedResultStatsByProject({ + project, +}: { + project: Project +}) { + const issuesRepo = new IssuesRepository(project.workspaceId) + const evalResultRepo = new EvaluationResultsV2Repository(project.workspaceId) + const issuesCount = await issuesRepo.countByProject({ project }) + const annotatedCount = + await evalResultRepo.failedManuallyAnnotatedCountByProject({ project }) + + return { + issuesCount, + annotatedCount, + } +} diff --git a/packages/core/src/repositories/evaluationResultsV2Repository.ts b/packages/core/src/repositories/evaluationResultsV2Repository.ts index a0db303753..f8a66901ab 100644 --- a/packages/core/src/repositories/evaluationResultsV2Repository.ts +++ b/packages/core/src/repositories/evaluationResultsV2Repository.ts @@ -50,6 +50,7 @@ import serializeProviderLog from '../services/providerLogs/serialize' import { CommitsRepository } from './commitsRepository' import { EvaluationsV2Repository } from './evaluationsV2Repository' import Repository from './repositoryV2' +import { Project } from '../schema/models/types/Project' const tt = getTableColumns(evaluationResultsV2) @@ -345,14 +346,14 @@ export class EvaluationResultsV2Repository extends Repository>'tokens')::bigint)` - .mapWith(Number) - .as('total_tokens') + .mapWith(Number) + .as('total_tokens') : sql`0`.mapWith(Number), totalCost: evaluation.type === EvaluationType.Llm ? sql`sum((${evaluationResultsV2.metadata}->>'cost')::bigint)` - .mapWith(Number) - .as('total_cost') + .mapWith(Number) + .as('total_cost') : sql`0`.mapWith(Number), } @@ -584,6 +585,23 @@ export class EvaluationResultsV2Repository extends Repository(result.count) } + async failedManuallyAnnotatedCountByProject({ project }: { project: Project }) { + const result = await this.db + .select({ count: count() }) + .from(evaluationResultsV2) + .innerJoin(commits, eq(commits.id, evaluationResultsV2.commitId)) + .where( + and( + this.scopeFilter, + eq(commits.projectId, project.id), + eq(evaluationResultsV2.hasPassed, false), + sql`${evaluationResultsV2.metadata}->>'reason' IS NOT NULL AND ${evaluationResultsV2.metadata}->>'reason' != ''`, + ), + ) + + return result[0]?.count ?? 0 + } + async selectForDocumentSuggestion({ commitId, evaluationUuid, diff --git a/packages/core/src/repositories/issuesRepository.ts b/packages/core/src/repositories/issuesRepository.ts index 9e7d90f0df..7d52575888 100644 --- a/packages/core/src/repositories/issuesRepository.ts +++ b/packages/core/src/repositories/issuesRepository.ts @@ -53,6 +53,17 @@ export class IssuesRepository extends Repository { return this.db.select(tt).from(issues).where(this.scopeFilter).$dynamic() } + async countByProject({ project }: { project: Project }) { + const result = await this.db + .select({ + count: sql`COUNT(*)::integer`, + }) + .from(issues) + .where(and(this.scopeFilter, eq(issues.projectId, project.id))) + + return result[0]?.count ?? 0 + } + async lock({ id, wait }: { id: number; wait?: boolean }) { // .for('no key update', { noWait: true }) is bugged in drizzle! // https://github.com/drizzle-team/drizzle-orm/issues/3554 @@ -224,8 +235,8 @@ export class IssuesRepository extends Repository { 'isResolved', ), isRegressed: sql`( - ${issues.resolvedAt} IS NOT NULL - AND ${issues.ignoredAt} IS NULL + ${issues.resolvedAt} IS NOT NULL + AND ${issues.ignoredAt} IS NULL AND ${subquery.lastSeenDate} > ${issues.resolvedAt} )`.as('isRegressed'), isEscalating: sql`( From 3bfe606e794cd4a17a3900155db78280e325dc17 Mon Sep 17 00:00:00 2001 From: andresgutgon Date: Fri, 14 Nov 2025 17:41:35 +0100 Subject: [PATCH 2/2] WIP --- .../IssuesOverviewCard/index.tsx} | 44 ++----------------- .../IssuesOverviewCard/issuesOverviewStore.ts | 36 +++++++++++++++ apps/web/src/services/routes/api.ts | 5 +++ 3 files changed, 45 insertions(+), 40 deletions(-) rename apps/web/src/{app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx => components/IssuesOverviewCard/index.tsx} (61%) create mode 100644 apps/web/src/components/IssuesOverviewCard/issuesOverviewStore.ts diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx b/apps/web/src/components/IssuesOverviewCard/index.tsx similarity index 61% rename from apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx rename to apps/web/src/components/IssuesOverviewCard/index.tsx index fe8a5b20a2..bfcd20f4ef 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/IssuesOverviewCard.tsx +++ b/apps/web/src/components/IssuesOverviewCard/index.tsx @@ -1,42 +1,15 @@ 'use client' -import { Button } from '@latitude-data/web-ui/atoms/Button' import { Text } from '@latitude-data/web-ui/atoms/Text' import { Icon } from '@latitude-data/web-ui/atoms/Icons' import { useCurrentProject } from '$/app/providers/ProjectProvider' -import { useCurrentCommit } from '$/app/providers/CommitProvider' -import { ROUTES } from '$/services/routes' -import { useRouter } from 'next/navigation' -import useSWR from 'swr' -import { IssuesOverviewResponse } from '$/app/api/projects/[projectId]/issues/overview/route' - -const fetcher = (url: string) => fetch(url).then((r) => r.json()) +import { useIssuesOverview } from './issuesOverviewStore' export function IssuesOverviewCard() { const { project } = useCurrentProject() - const { commit } = useCurrentCommit() - const router = useRouter() - - const { data, isLoading } = useSWR( - `/api/projects/${project.id}/issues/overview`, - fetcher, - ) - - if (isLoading || !data) return null - if (data.totalAnnotations === 0) return null - - const progress = data.totalAnnotations > 0 - ? (data.issuesWithAnnotations / data.totalAnnotations) * 100 - : 0 - - const handleReviewIssues = () => { - router.push( - ROUTES.projects - .detail({ id: project.id }) - .commits.detail({ uuid: commit.uuid }) - .issues.root, - ) - } + const { data, isLoading } = useIssuesOverview({ projectId: project.id }) + // TODO: Implement + const progress = isLoading ? 0 : 0 return (
@@ -47,15 +20,6 @@ export function IssuesOverviewCard() { Keep annotating to discover more issues.
-
diff --git a/apps/web/src/components/IssuesOverviewCard/issuesOverviewStore.ts b/apps/web/src/components/IssuesOverviewCard/issuesOverviewStore.ts new file mode 100644 index 0000000000..7c4ca031a1 --- /dev/null +++ b/apps/web/src/components/IssuesOverviewCard/issuesOverviewStore.ts @@ -0,0 +1,36 @@ +import { IssuesOverviewResponse } from '$/app/api/projects/[projectId]/issues/overview/route' +import useFetcher from '$/hooks/useFetcher' +import { ROUTES } from '$/services/routes' +import { useMemo } from 'react' +import useSWR, { SWRConfiguration } from 'swr' + +const EMPTY_OVERVIEW: IssuesOverviewResponse = { + issuesCount: 0, + annotatedCount: 0, +} +export function useIssuesOverview( + { + projectId, + }: { + projectId: number + }, + swrConfig?: SWRConfiguration, +) { + const route = ROUTES.api.projects.detail(projectId).issues.overview.root + const fetcher = useFetcher( + route, + ) + const { data = EMPTY_OVERVIEW, isLoading } = useSWR( + ['issues-overview', projectId], + fetcher, + swrConfig, + ) + + return useMemo( + () => ({ + data, + isLoading, + }), + [data, isLoading], + ) +} diff --git a/apps/web/src/services/routes/api.ts b/apps/web/src/services/routes/api.ts index 1b40a89c55..bccc4e5eb0 100644 --- a/apps/web/src/services/routes/api.ts +++ b/apps/web/src/services/routes/api.ts @@ -121,6 +121,11 @@ export const API_ROUTES = { stats: { root: `${projectRoot}/stats`, }, + issues: { + overview: { + root: `${projectRoot}/issues/overview`, + }, + }, commits: { root: `${projectRoot}/commits`, detail: (commitUuid: string) => ({