diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx index 6dd9da0189..0701587883 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/runs/_components/RunsPage.tsx @@ -29,6 +29,7 @@ import { AppLocalStorage, useLocalStorage, } from '@latitude-data/web-ui/hooks/useLocalStorage' +import { IssuesOverviewCard } from './IssuesOverviewCard' function sumCounts( counts: Record | undefined, @@ -284,6 +285,7 @@ export function RunsPage({ /> ) : (
+
No run selected
diff --git a/apps/web/src/app/api/projects/[projectId]/issues/overview/route.ts b/apps/web/src/app/api/projects/[projectId]/issues/overview/route.ts new file mode 100644 index 0000000000..da64b0a32b --- /dev/null +++ b/apps/web/src/app/api/projects/[projectId]/issues/overview/route.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' +import { authHandler } from '$/middlewares/authHandler' +import { errorHandler } from '$/middlewares/errorHandler' +import { ProjectsRepository } from '@latitude-data/core/repositories' +import { Workspace } from '@latitude-data/core/schema/models/types/Workspace' +import { NextRequest, NextResponse } from 'next/server' +import { annotatedResultStatsByProject } from '@latitude-data/core/data-access/issues/annotatedResultStatsByProject' + +const paramsSchema = z.object({ + projectId: z.coerce.number(), +}) + +export type IssuesOverviewResponse = Awaited< + ReturnType +> + +export const GET = errorHandler( + authHandler( + async ( + _request: NextRequest, + { + params, + workspace, + }: { + params: { + projectId: string + } + workspace: Workspace + }, + ) => { + const { projectId } = paramsSchema.parse({ + projectId: params.projectId, + }) + const projectsRepo = new ProjectsRepository(workspace.id) + const project = await projectsRepo.find(projectId).then((r) => r.unwrap()) + const stats = await annotatedResultStatsByProject({ project }) + + return NextResponse.json(stats, { status: 200 }) + }, + ), +) diff --git a/apps/web/src/components/IssuesOverviewCard/index.tsx b/apps/web/src/components/IssuesOverviewCard/index.tsx new file mode 100644 index 0000000000..bfcd20f4ef --- /dev/null +++ b/apps/web/src/components/IssuesOverviewCard/index.tsx @@ -0,0 +1,57 @@ +'use client' + +import { Text } from '@latitude-data/web-ui/atoms/Text' +import { Icon } from '@latitude-data/web-ui/atoms/Icons' +import { useCurrentProject } from '$/app/providers/ProjectProvider' +import { useIssuesOverview } from './issuesOverviewStore' + +export function IssuesOverviewCard() { + const { project } = useCurrentProject() + const { data, isLoading } = useIssuesOverview({ projectId: project.id }) + // TODO: Implement + const progress = isLoading ? 0 : 0 + + return ( +
+
+
+ Issues overview is now available! + + Keep annotating to discover more issues. + +
+
+ +
+
+ {/* Background bar */} +
+ {/* Progress bar */} +
+
+
+ + {/* Lightbulb icon */} +
+
+ +
+ +
+ +
+
+ +
+
+ Insight discovery ยท + {data.issuesWithAnnotations} +
+ {data.totalAnnotations} +
+
+ ) +} diff --git a/apps/web/src/components/IssuesOverviewCard/issuesOverviewStore.ts b/apps/web/src/components/IssuesOverviewCard/issuesOverviewStore.ts new file mode 100644 index 0000000000..7c4ca031a1 --- /dev/null +++ b/apps/web/src/components/IssuesOverviewCard/issuesOverviewStore.ts @@ -0,0 +1,36 @@ +import { IssuesOverviewResponse } from '$/app/api/projects/[projectId]/issues/overview/route' +import useFetcher from '$/hooks/useFetcher' +import { ROUTES } from '$/services/routes' +import { useMemo } from 'react' +import useSWR, { SWRConfiguration } from 'swr' + +const EMPTY_OVERVIEW: IssuesOverviewResponse = { + issuesCount: 0, + annotatedCount: 0, +} +export function useIssuesOverview( + { + projectId, + }: { + projectId: number + }, + swrConfig?: SWRConfiguration, +) { + const route = ROUTES.api.projects.detail(projectId).issues.overview.root + const fetcher = useFetcher( + route, + ) + const { data = EMPTY_OVERVIEW, isLoading } = useSWR( + ['issues-overview', projectId], + fetcher, + swrConfig, + ) + + return useMemo( + () => ({ + data, + isLoading, + }), + [data, isLoading], + ) +} diff --git a/apps/web/src/services/routes/api.ts b/apps/web/src/services/routes/api.ts index 1b40a89c55..bccc4e5eb0 100644 --- a/apps/web/src/services/routes/api.ts +++ b/apps/web/src/services/routes/api.ts @@ -121,6 +121,11 @@ export const API_ROUTES = { stats: { root: `${projectRoot}/stats`, }, + issues: { + overview: { + root: `${projectRoot}/issues/overview`, + }, + }, commits: { root: `${projectRoot}/commits`, detail: (commitUuid: string) => ({ diff --git a/packages/constants/src/issues/constants.ts b/packages/constants/src/issues/constants.ts index 94f1c3a7d3..227e19ee3e 100644 --- a/packages/constants/src/issues/constants.ts +++ b/packages/constants/src/issues/constants.ts @@ -97,3 +97,7 @@ export const ISSUE_JOBS_MAX_ATTEMPTS = 3 export const ISSUE_JOBS_GENERATE_DETAILS_THROTTLE = 6 * 60 * 60 * 1000 // 6 hours export const ISSUE_JOBS_MERGE_COMMON_THROTTLE = 24 * 60 * 60 * 1000 // 1 day export const ISSUE_JOBS_DISCOVER_RESULT_DELAY = 60 * 1000 // 1 minute + +// This is the minimum number of manually annotated evaluation results required +// for automatic issue creation to be enabled for a project. +export const MINIMUM_ANNOTATED_THRESHOLD = 10 diff --git a/packages/core/src/data-access/issues/annotatedResultStatsByProject.ts b/packages/core/src/data-access/issues/annotatedResultStatsByProject.ts new file mode 100644 index 0000000000..26c1cc4a53 --- /dev/null +++ b/packages/core/src/data-access/issues/annotatedResultStatsByProject.ts @@ -0,0 +1,22 @@ +import { + EvaluationResultsV2Repository, + IssuesRepository, +} from '../../repositories' +import { Project } from '../../schema/models/types/Project' + +export async function annotatedResultStatsByProject({ + project, +}: { + project: Project +}) { + const issuesRepo = new IssuesRepository(project.workspaceId) + const evalResultRepo = new EvaluationResultsV2Repository(project.workspaceId) + const issuesCount = await issuesRepo.countByProject({ project }) + const annotatedCount = + await evalResultRepo.failedManuallyAnnotatedCountByProject({ project }) + + return { + issuesCount, + annotatedCount, + } +} diff --git a/packages/core/src/repositories/evaluationResultsV2Repository.ts b/packages/core/src/repositories/evaluationResultsV2Repository.ts index a0db303753..f8a66901ab 100644 --- a/packages/core/src/repositories/evaluationResultsV2Repository.ts +++ b/packages/core/src/repositories/evaluationResultsV2Repository.ts @@ -50,6 +50,7 @@ import serializeProviderLog from '../services/providerLogs/serialize' import { CommitsRepository } from './commitsRepository' import { EvaluationsV2Repository } from './evaluationsV2Repository' import Repository from './repositoryV2' +import { Project } from '../schema/models/types/Project' const tt = getTableColumns(evaluationResultsV2) @@ -345,14 +346,14 @@ export class EvaluationResultsV2Repository extends Repository>'tokens')::bigint)` - .mapWith(Number) - .as('total_tokens') + .mapWith(Number) + .as('total_tokens') : sql`0`.mapWith(Number), totalCost: evaluation.type === EvaluationType.Llm ? sql`sum((${evaluationResultsV2.metadata}->>'cost')::bigint)` - .mapWith(Number) - .as('total_cost') + .mapWith(Number) + .as('total_cost') : sql`0`.mapWith(Number), } @@ -584,6 +585,23 @@ export class EvaluationResultsV2Repository extends Repository(result.count) } + async failedManuallyAnnotatedCountByProject({ project }: { project: Project }) { + const result = await this.db + .select({ count: count() }) + .from(evaluationResultsV2) + .innerJoin(commits, eq(commits.id, evaluationResultsV2.commitId)) + .where( + and( + this.scopeFilter, + eq(commits.projectId, project.id), + eq(evaluationResultsV2.hasPassed, false), + sql`${evaluationResultsV2.metadata}->>'reason' IS NOT NULL AND ${evaluationResultsV2.metadata}->>'reason' != ''`, + ), + ) + + return result[0]?.count ?? 0 + } + async selectForDocumentSuggestion({ commitId, evaluationUuid, diff --git a/packages/core/src/repositories/issuesRepository.ts b/packages/core/src/repositories/issuesRepository.ts index 9e7d90f0df..7d52575888 100644 --- a/packages/core/src/repositories/issuesRepository.ts +++ b/packages/core/src/repositories/issuesRepository.ts @@ -53,6 +53,17 @@ export class IssuesRepository extends Repository { return this.db.select(tt).from(issues).where(this.scopeFilter).$dynamic() } + async countByProject({ project }: { project: Project }) { + const result = await this.db + .select({ + count: sql`COUNT(*)::integer`, + }) + .from(issues) + .where(and(this.scopeFilter, eq(issues.projectId, project.id))) + + return result[0]?.count ?? 0 + } + async lock({ id, wait }: { id: number; wait?: boolean }) { // .for('no key update', { noWait: true }) is bugged in drizzle! // https://github.com/drizzle-team/drizzle-orm/issues/3554 @@ -224,8 +235,8 @@ export class IssuesRepository extends Repository { 'isResolved', ), isRegressed: sql`( - ${issues.resolvedAt} IS NOT NULL - AND ${issues.ignoredAt} IS NULL + ${issues.resolvedAt} IS NOT NULL + AND ${issues.ignoredAt} IS NULL AND ${subquery.lastSeenDate} > ${issues.resolvedAt} )`.as('isRegressed'), isEscalating: sql`(