Connect and disconnect issues from evals (#1942)

learningbizz · web-flow · commit 340b3ab8beb1 · 2025-11-19T15:58:30.000+01:00
* WIP - connect and disconnect issues from evals

* fix PR comments + move eval issue actions to eval results issue actions

* Add new button

* fix linter

* remove double button

* remove consolelog
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/assignIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/assignIssue.ts
@@ -6,7 +6,7 @@ import {
 } from '@latitude-data/core/repositories'
 import { assignEvaluationResultV2ToIssue } from '@latitude-data/core/services/evaluationsV2/results/assign'
 import { z } from 'zod'
-import { withEvaluation, withEvaluationSchema } from '../../procedures'
+import { withEvaluation, withEvaluationSchema } from '$/actions/procedures'
 
 export const assignIssueAction = withEvaluation
   .inputSchema(
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/createIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/createIssue.ts
@@ -3,7 +3,7 @@
 import { EvaluationResultsV2Repository } from '@latitude-data/core/repositories'
 import { assignEvaluationResultV2ToIssue } from '@latitude-data/core/services/evaluationsV2/results/assign'
 import { z } from 'zod'
-import { withEvaluation, withEvaluationSchema } from '../../procedures'
+import { withEvaluation, withEvaluationSchema } from '$/actions/procedures'
 
 export const createIssueAction = withEvaluation
   .inputSchema(
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/generateIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/generateIssue.ts
@@ -3,7 +3,7 @@
 import { EvaluationResultsV2Repository } from '@latitude-data/core/repositories'
 import { generateIssue } from '@latitude-data/core/services/issues/generate'
 import { z } from 'zod'
-import { withEvaluation, withEvaluationSchema } from '../../procedures'
+import { withEvaluation, withEvaluationSchema } from '$/actions/procedures'
 
 export const generateIssueAction = withEvaluation
   .inputSchema(
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/ignoreIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/ignoreIssue.ts
@@ -3,7 +3,7 @@
 import { IssuesRepository } from '@latitude-data/core/repositories'
 import { ignoreIssue } from '@latitude-data/core/services/issues/ignore'
 import { z } from 'zod'
-import { withCommit, withCommitSchema } from '../../procedures'
+import { withCommit, withCommitSchema } from '$/actions/procedures'
 
 export const ignoreIssueAction = withCommit
   .inputSchema(
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/resolveIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/resolveIssue.ts
@@ -3,7 +3,7 @@
 import { IssuesRepository } from '@latitude-data/core/repositories'
 import { resolveIssue } from '@latitude-data/core/services/issues/resolve'
 import { z } from 'zod'
-import { withCommit, withCommitSchema } from '../../procedures'
+import { withCommit, withCommitSchema } from '$/actions/procedures'
 
 export const resolveIssueAction = withCommit
   .inputSchema(
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/unAssignIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/unAssignIssue.ts
@@ -6,7 +6,7 @@ import {
 } from '@latitude-data/core/repositories'
 import { unassignEvaluationResultV2FromIssue } from '@latitude-data/core/services/evaluationsV2/results/unassign'
 import { z } from 'zod'
-import { withEvaluation, withEvaluationSchema } from '../../procedures'
+import { withEvaluation, withEvaluationSchema } from '$/actions/procedures'
 
 export const unAssignIssueAction = withEvaluation
   .inputSchema(
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/unignoreIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/unignoreIssue.ts
@@ -3,7 +3,7 @@
 import { IssuesRepository } from '@latitude-data/core/repositories'
 import { unignoreIssue } from '@latitude-data/core/services/issues/unignore'
 import { z } from 'zod'
-import { withCommit, withCommitSchema } from '../../procedures'
+import { withCommit, withCommitSchema } from '$/actions/procedures'
 
 export const unignoreIssueAction = withCommit
   .inputSchema(
diff --git a/apps/web/src/actions/evaluationsV2/results/issues/unresolveIssue.ts b/apps/web/src/actions/evaluationsV2/results/issues/unresolveIssue.ts
@@ -3,7 +3,7 @@
 import { IssuesRepository } from '@latitude-data/core/repositories'
 import { unresolveIssue } from '@latitude-data/core/services/issues/unresolve'
 import { z } from 'zod'
-import { withCommit, withCommitSchema } from '../../procedures'
+import { withCommit, withCommitSchema } from '$/actions/procedures'
 
 export const unresolveIssueAction = withCommit
   .inputSchema(
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssueDetailPanel/Evaluation/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssueDetailPanel/Evaluation/index.tsx
@@ -0,0 +1,136 @@
+'use client'
+
+import { useCurrentCommit } from '$/app/providers/CommitProvider'
+import { Text } from '@latitude-data/web-ui/atoms/Text'
+import { useCurrentProject } from '$/app/providers/ProjectProvider'
+import { useEvaluationsV2 } from '$/stores/evaluationsV2'
+import React, { useCallback, useMemo } from 'react'
+import { Select } from '@latitude-data/web-ui/atoms/Select'
+import { Icon } from '@latitude-data/web-ui/atoms/Icons'
+import { getEvaluationMetricSpecification } from '$/components/evaluations'
+import { Button } from '@latitude-data/web-ui/atoms/Button'
+import { Skeleton } from '@latitude-data/web-ui/atoms/Skeleton'
+import { Issue } from '@latitude-data/core/schema/models/types/Issue'
+
+export function IssueEvaluation({ issue }: { issue: Issue }) {
+  const { project } = useCurrentProject()
+  const { commit } = useCurrentCommit()
+
+  const {
+    data: evaluations,
+    isLoading: isLoadingEvaluations,
+    updateEvaluation,
+    isUpdatingEvaluation,
+    //    isGeneratingEvaluationFromIssue,
+    //    generateEvaluationFromIssue,
+  } = useEvaluationsV2({
+    project: project,
+    commit: commit,
+    document: {
+      commitId: commit.id,
+      documentUuid: issue.documentUuid,
+    },
+  })
+
+  const evaluationsThatCanBeAttachedToIssues = useMemo(
+    () =>
+      evaluations.filter(
+        (e) =>
+          !getEvaluationMetricSpecification(e).requiresExpectedOutput &&
+          getEvaluationMetricSpecification(e).supportsLiveEvaluation,
+      ),
+    [evaluations],
+  )
+
+  const evaluationWithIssue = useMemo(
+    () =>
+      evaluationsThatCanBeAttachedToIssues.find((e) => e.issueId === issue.id),
+    [evaluationsThatCanBeAttachedToIssues, issue.id],
+  )
+
+  const setIssueForNewEvaluation = useCallback(
+    (newEvaluationUuid: string) => {
+      // If the issue already had an eval attached, remove it
+      if (evaluationWithIssue) {
+        updateEvaluation({
+          evaluationUuid: evaluationWithIssue.uuid,
+          issueId: null,
+        })
+      }
+      // If a new evaluation is selected, attach it to the issue
+      if (newEvaluationUuid) {
+        updateEvaluation({
+          evaluationUuid: newEvaluationUuid,
+          issueId: issue.id,
+        })
+      }
+    },
+    [evaluationWithIssue, issue.id, updateEvaluation],
+  )
+
+  if (isLoadingEvaluations) {
+    return (
+      <div className='grid grid-cols-2 gap-x-4 items-center'>
+        <Text.H5 color='foregroundMuted'>Evaluation</Text.H5>
+        <Skeleton className='w-full h-10' />
+      </div>
+    )
+  }
+
+  if (evaluationWithIssue) {
+    return (
+      <div className='grid grid-cols-2 gap-x-4 items-center'>
+        <Text.H5 color='foregroundMuted'>Evaluation</Text.H5>
+        <div>
+          <Select
+            badgeLabel
+            align='end'
+            searchable
+            side='bottom'
+            removable
+            name='evaluation'
+            options={evaluations.map((e) => ({
+              label: e.name,
+              value: e.uuid,
+              icon: <Icon name={getEvaluationMetricSpecification(e).icon} />,
+            }))}
+            value={evaluationWithIssue?.uuid}
+            disabled={isUpdatingEvaluation}
+            loading={isUpdatingEvaluation}
+            onChange={setIssueForNewEvaluation}
+            // footerAction={{
+            //   label: 'Generate new evaluation',
+            //   icon: 'wandSparkles',
+            //   onClick: () => undefined,
+            // }} // TODO(evaluation-generator): Remove this once we have a way to generate evaluations from issues
+          />
+        </div>
+      </div>
+    )
+  }
+
+  return (
+    <div className='grid grid-cols-2 gap-x-4 items-center'>
+      <Text.H5 color='foregroundMuted'>Evaluation</Text.H5>
+      <div>
+        <Button
+          variant='primaryMuted'
+          iconProps={{
+            name: 'wandSparkles',
+            color: 'primary',
+            placement: 'left',
+          }}
+          disabled={true} // TODO(evaluation-generator): Remove this once we have a way to generate evaluations from issues
+          onClick={() => {
+            //   e.stopPropagation()
+            //   generateEvaluationFromIssue({ issueId: issue.id })
+          }} // TODO(evaluation-generator): Uncomment this once we have a way to generate evaluations from issues
+          // disabled: isGeneratingEvaluationFromIssue,
+          // isLoading: isGeneratingEvaluationFromIssue,
+        >
+          Generate
+        </Button>
+      </div>
+    </div>
+  )
+}
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssueDetailPanel/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssueDetailPanel/index.tsx
@@ -1,4 +1,4 @@
-import { RefObject, useRef } from 'react'
+import { RefObject, useMemo, useRef } from 'react'
 import { usePanelDomRef } from '@latitude-data/web-ui/atoms/SplitPane'
 import { Text } from '@latitude-data/web-ui/atoms/Text'
 import { StickyOffset, useStickyNested } from '$/hooks/useStickyNested'
@@ -9,6 +9,9 @@ import { IssueDetails } from './IssueDetails'
 import { IssueItemActions } from '../IssueItemActions'
 import { Button } from '@latitude-data/web-ui/atoms/Button'
 import { Tooltip } from '@latitude-data/web-ui/atoms/Tooltip'
+import useFeature from '$/stores/useFeature'
+import { IssueEvaluation } from './Evaluation'
+import { Separator } from '@latitude-data/web-ui/atoms/Separator'
 
 export function IssuesDetailPanel({
   stickyRef,
@@ -24,6 +27,13 @@ export function IssuesDetailPanel({
   offset: StickyOffset
 }) {
   const ref = useRef<HTMLDivElement>(null)
+  const { isEnabled: itIs, isLoading: isLoadingEvaluationGeneratorEnabled } =
+    useFeature('evaluationGenerator')
+
+  const isEvaluationGeneratorEnabled = useMemo(
+    () => itIs && !isLoadingEvaluationGeneratorEnabled,
+    [itIs, isLoadingEvaluationGeneratorEnabled],
+  )
 
   const scrollableArea = usePanelDomRef({ selfRef: ref.current })
   const beacon = stickyRef?.current
@@ -59,7 +69,9 @@ export function IssuesDetailPanel({
         </div>
       </DetailsPanel.Header>
       <DetailsPanel.Body>
-        <div className='flex flex-col gap-y-6'>
+        <div className='flex flex-col gap-y-6 pt-1'>
+          {isEvaluationGeneratorEnabled && <IssueEvaluation issue={issue} />}
+          {isEvaluationGeneratorEnabled && <Separator variant='dashed' />}
           <IssueDetails issue={issue} />
           <TracesList issue={issue} />
         </div>
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssueItemActions/useActions.ts b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssueItemActions/useActions.ts
@@ -1,9 +1,9 @@
 import { useCallback, useMemo, useState } from 'react'
 import { useSWRConfig } from 'swr'
-import { ignoreIssueAction } from '$/actions/evaluationsV2/issues/ignoreIssue'
-import { resolveIssueAction } from '$/actions/evaluationsV2/issues/resolveIssue'
-import { unignoreIssueAction } from '$/actions/evaluationsV2/issues/unignoreIssue'
-import { unresolveIssueAction } from '$/actions/evaluationsV2/issues/unresolveIssue'
+import { ignoreIssueAction } from '$/actions/evaluationsV2/results/issues/ignoreIssue'
+import { resolveIssueAction } from '$/actions/evaluationsV2/results/issues/resolveIssue'
+import { unignoreIssueAction } from '$/actions/evaluationsV2/results/issues/unignoreIssue'
+import { unresolveIssueAction } from '$/actions/evaluationsV2/results/issues/unresolveIssue'
 import useLatitudeAction from '$/hooks/useLatitudeAction'
 import { useCurrentCommit } from '$/app/providers/CommitProvider'
 import { useCurrentProject } from '$/app/providers/ProjectProvider'
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssuesTable/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/issues/_components/IssuesTable/index.tsx
@@ -107,6 +107,14 @@ export function IssuesTable({
     onSelectChange(undefined)
   }, [onSelectChange])
 
+  // TODO(evaluation-generation): remove and use when we have the design for evaluations in the issues table
+  // const { data: evaluations, isLoading: isLoadingEvaluations } =
+  // useIssueEvaluations({
+  //   projectId: project.id,
+  //   commitUuid: commit.uuid,
+  //   documentUuids: issues?.map((issue) => issue.documentUuid) ?? [],
+  // })
+
   if (noData) {
     return (
       <TableBlankSlate description='No issues discovered in this project yet' />
diff --git a/apps/web/src/app/api/projects/[projectId]/commits/[commitUuid]/issues/evaluations/route.ts b/apps/web/src/app/api/projects/[projectId]/commits/[commitUuid]/issues/evaluations/route.ts
@@ -0,0 +1,56 @@
+import { authHandler } from '$/middlewares/authHandler'
+import { errorHandler } from '$/middlewares/errorHandler'
+import { EvaluationV2 } from '@latitude-data/core/constants'
+import { EvaluationsV2Repository } from '@latitude-data/core/repositories/evaluationsV2Repository'
+import { Workspace } from '@latitude-data/core/schema/models/types/Workspace'
+import { NextRequest, NextResponse } from 'next/server'
+import { z } from 'zod'
+
+const paramsSchema = z.object({
+  projectId: z.coerce.number(),
+  commitUuid: z.string(),
+})
+
+export const GET = errorHandler(
+  authHandler(
+    async (
+      request: NextRequest,
+      {
+        params,
+        workspace,
+      }: {
+        params: {
+          projectId: string
+          commitUuid: string
+        }
+        workspace: Workspace
+      },
+    ) => {
+      const { projectId, commitUuid } = paramsSchema.parse({
+        projectId: params.projectId,
+        commitUuid: params.commitUuid,
+      })
+
+      const documentUuids =
+        request.nextUrl.searchParams
+          .get('documentUuids')
+          ?.split(',')
+          .map((uuid) => uuid.trim()) || []
+
+      const evaluationsRepository = new EvaluationsV2Repository(workspace.id)
+      let evaluations: EvaluationV2[] = []
+      for (const documentUuid of documentUuids) {
+        const evaluationsForDocument = await evaluationsRepository
+          .listAtCommitByDocument({
+            projectId,
+            commitUuid,
+            documentUuid,
+          })
+          .then((r) => r.unwrap())
+        evaluations = evaluations.concat(evaluationsForDocument)
+      }
+
+      return NextResponse.json(evaluations, { status: 200 })
+    },
+  ),
+)
diff --git a/apps/web/src/services/routes/api.ts b/apps/web/src/services/routes/api.ts
@@ -140,6 +140,7 @@ export const API_ROUTES = {
               root: `${projectRoot}/commits/${commitUuid}/issues`,
               search: `${projectRoot}/commits/${commitUuid}/issues/search`,
               histograms: `${projectRoot}/commits/${commitUuid}/issues/histograms`,
+              evaluations: `${projectRoot}/commits/${commitUuid}/issues/evaluations`,
               detail: (issueId: number) => {
                 return {
                   root: `${projectRoot}/commits/${commitUuid}/issues/${issueId}`,
diff --git a/apps/web/src/stores/issues/evaluations.ts b/apps/web/src/stores/issues/evaluations.ts
@@ -0,0 +1,31 @@
+'use client'
+
+import useFetcher from '$/hooks/useFetcher'
+import { ROUTES } from '$/services/routes'
+import useSWR, { SWRConfiguration } from 'swr'
+import { EvaluationV2 } from '@latitude-data/core/constants'
+
+export function useIssueEvaluations(
+  {
+    projectId,
+    commitUuid,
+    documentUuids,
+  }: {
+    projectId: number
+    commitUuid: string
+    documentUuids: string[]
+  },
+  opts?: SWRConfiguration,
+) {
+  const route = ROUTES.api.projects.detail(projectId).commits.detail(commitUuid)
+    .issues.evaluations
+  const fetcher = useFetcher<EvaluationV2[]>(route, {
+    searchParams: { documentUuids: documentUuids.join(',') },
+  })
+  const { data, isLoading } = useSWR<EvaluationV2[]>(
+    ['issueEvaluations', projectId, commitUuid, documentUuids],
+    fetcher,
+    opts,
+  )
+  return { data, isLoading }
+}
diff --git a/apps/web/src/stores/issues/issue.ts b/apps/web/src/stores/issues/issue.ts
@@ -1,7 +1,7 @@
-import { assignIssueAction } from '$/actions/evaluationsV2/issues/assignIssue'
-import { createIssueAction } from '$/actions/evaluationsV2/issues/createIssue'
-import { generateIssueAction } from '$/actions/evaluationsV2/issues/generateIssue'
-import { unAssignIssueAction } from '$/actions/evaluationsV2/issues/unAssignIssue'
+import { assignIssueAction } from '$/actions/evaluationsV2/results/issues/assignIssue'
+import { createIssueAction } from '$/actions/evaluationsV2/results/issues/createIssue'
+import { generateIssueAction } from '$/actions/evaluationsV2/results/issues/generateIssue'
+import { unAssignIssueAction } from '$/actions/evaluationsV2/results/issues/unAssignIssue'
 import useFetcher from '$/hooks/useFetcher'
 import useLatitudeAction from '$/hooks/useLatitudeAction'
 import { ROUTES } from '$/services/routes'
diff --git a/packages/core/src/services/evaluationsV2/update.ts b/packages/core/src/services/evaluationsV2/update.ts
@@ -120,7 +120,7 @@ export async function updateEvaluationV2<
           evaluationVersions.commitId,
           evaluationVersions.evaluationUuid,
         ],
-        set: { ...settings, ...options, updatedAt: new Date() },
+        set: { ...settings, ...options, updatedAt: new Date(), issueId },
       })
       .returning()
       .then((r) => r[0]!)
diff --git a/packages/core/src/services/evaluationsV2/validate.test.ts b/packages/core/src/services/evaluationsV2/validate.test.ts
diff --git a/packages/core/src/services/evaluationsV2/validate.ts b/packages/core/src/services/evaluationsV2/validate.ts
diff --git a/packages/web-ui/package.json b/packages/web-ui/package.json
diff --git a/packages/web-ui/src/ds/atoms/Icons/index.tsx b/packages/web-ui/src/ds/atoms/Icons/index.tsx
