Evaluation output configuration improvements (#1849)

* Evaluation output configuration improvements

* remove console.log

Author: neoxelox

apps/gateway/src/routes/api/v3/conversations/annotate/annotate.handler.test.ts

@@ -54,9 +54,16 @@ describe('POST /conversations/:conversationUuid/evaluations/:evaluationUuid/anno
 
   // Default evaluation configuration
   const DEFAULT_EVALUATION_CONFIG = {
+    reverseScale: false,
+    actualOutput: {
+      messageSelection: 'last' as const,
+      parsingFormat: 'string' as const,
+    },
+    expectedOutput: {
+      parsingFormat: 'string' as const,
+    },
     minRating: 1,
     maxRating: 5,
-    reverseScale: false,
   }
 
   type TestSetupOptions = {

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/(withTabs)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Playground/EvaluationParams/HistoryLogParams/useSerializedLogs.ts

@@ -1,17 +1,17 @@
+import { useCurrentCommit } from '$/app/providers/CommitProvider'
+import { useCurrentProject } from '$/app/providers/ProjectProvider'
 import { useDefaultLogFilterOptions } from '$/hooks/logFilters/useDefaultLogFilterOptions'
 import useDocumentLogWithPaginationPosition, {
   LogWithPosition,
 } from '$/stores/documentLogWithPaginationPosition'
 import useEvaluatedDocumentLogs from '$/stores/evaluatedDocumentLogs'
 import useDocumentLogsPagination from '$/stores/useDocumentLogsPagination'
-import { useCurrentCommit } from '$/app/providers/CommitProvider'
-import { useCurrentProject } from '$/app/providers/ProjectProvider'
-import { useCallback, useMemo, useState } from 'react'
 import {
   ActualOutputConfiguration,
   EvaluatedDocumentLog,
 } from '@latitude-data/core/constants'
 import { DocumentVersion } from '@latitude-data/core/schema/models/types/DocumentVersion'
+import { useCallback, useMemo, useState } from 'react'
 
 const ONLY_ONE_PAGE = '1'
 
@@ -23,7 +23,7 @@ export function useSerializedLogs({
   logUuid,
 }: {
   document: DocumentVersion
-  configuration?: ActualOutputConfiguration
+  configuration: ActualOutputConfiguration
   onHistoryFetched?: OnHistoryFetchedFn
   logUuid?: string
 }) {

apps/web/src/app/api/projects/[projectId]/documents/[documentUuid]/evaluatedLogs/route.ts

@@ -1,30 +1,30 @@
 import { authHandler } from '$/middlewares/authHandler'
 import { errorHandler } from '$/middlewares/errorHandler'
+import {
+  ActualOutputConfiguration,
+  DocumentLog,
+  EvaluatedDocumentLog,
+} from '@latitude-data/core/constants'
 import { findLastProviderLogFromDocumentLogUuid } from '@latitude-data/core/data-access/providerLogs'
+import {
+  buildConversation,
+  formatConversation,
+} from '@latitude-data/core/helpers'
 import { UnprocessableEntityError } from '@latitude-data/core/lib/errors'
 import {
   DocumentVersionsRepository,
   ProviderLogsRepository,
 } from '@latitude-data/core/repositories'
+import { ProviderLog } from '@latitude-data/core/schema/models/types/ProviderLog'
+import { Workspace } from '@latitude-data/core/schema/models/types/Workspace'
+import { ProviderLogDto } from '@latitude-data/core/schema/types'
 import { computeDocumentLogs } from '@latitude-data/core/services/documentLogs/computeDocumentLogs'
 import { parseApiDocumentLogParams } from '@latitude-data/core/services/documentLogs/logsFilterUtils/parseApiLogFilterParams'
 import { serializeAggregatedProviderLog } from '@latitude-data/core/services/documentLogs/serialize'
-import { buildProviderLogResponse } from '@latitude-data/core/services/providerLogs/buildResponse'
 import { extractActualOutput } from '@latitude-data/core/services/evaluationsV2/outputs/extract'
+import { buildProviderLogResponse } from '@latitude-data/core/services/providerLogs/buildResponse'
 import { NextRequest, NextResponse } from 'next/server'
-import {
-  ActualOutputConfiguration,
-  DocumentLog,
-  EvaluatedDocumentLog,
-} from '@latitude-data/core/constants'
-import {
-  buildConversation,
-  formatConversation,
-} from '@latitude-data/core/helpers'
-import { ProviderLogDto } from '@latitude-data/core/schema/types'
 
-import { Workspace } from '@latitude-data/core/schema/models/types/Workspace'
-import { ProviderLog } from '@latitude-data/core/schema/models/types/ProviderLog'
 export const GET = errorHandler(
   authHandler(
     async (
@@ -112,7 +112,7 @@ async function serializeEvaluatedDocumentLog({
 }: {
   documentLog: DocumentLog
   providerLogs: ProviderLog[]
-  configuration?: ActualOutputConfiguration
+  configuration: ActualOutputConfiguration
 }): Promise<EvaluatedDocumentLog> {
   const providerLog = (await findLastProviderLogFromDocumentLogUuid(
     documentLog.uuid,

apps/web/src/components/evaluations/ConfigurationForm.tsx

@@ -1,6 +1,16 @@
 import { useSerializedLogs } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/(withTabs)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Playground/EvaluationParams/HistoryLogParams/useSerializedLogs'
+import { useCurrentCommit } from '$/app/providers/CommitProvider'
 import { useCurrentDocument } from '$/app/providers/DocumentProvider'
+import { useCurrentProject } from '$/app/providers/ProjectProvider'
+import { MessageList, MessageListSkeleton } from '$/components/ChatWrapper'
 import { ROUTES } from '$/services/routes'
+import {
+  ACCESSIBLE_OUTPUT_FORMATS,
+  ActualOutputConfiguration,
+  EvaluationMetric,
+  EvaluationType,
+  baseEvaluationConfiguration,
+} from '@latitude-data/core/constants'
 import { Alert } from '@latitude-data/web-ui/atoms/Alert'
 import { Button } from '@latitude-data/web-ui/atoms/Button'
 import { FormFieldGroup } from '@latitude-data/web-ui/atoms/FormFieldGroup'
@@ -9,44 +19,36 @@ import { LineSeparator } from '@latitude-data/web-ui/atoms/LineSeparator'
 import { Select } from '@latitude-data/web-ui/atoms/Select'
 import { Skeleton } from '@latitude-data/web-ui/atoms/Skeleton'
 import { Text } from '@latitude-data/web-ui/atoms/Text'
-import { MessageList, MessageListSkeleton } from '$/components/ChatWrapper'
 import { SelectableSwitch } from '@latitude-data/web-ui/molecules/SelectableSwitch'
 import { ClickToCopyUuid } from '@latitude-data/web-ui/organisms/ClickToCopyUuid'
-import { useCurrentCommit } from '$/app/providers/CommitProvider'
-import { useCurrentProject } from '$/app/providers/ProjectProvider'
 import { useRouter } from 'next/navigation'
 import { useEffect, useMemo, useState } from 'react'
 import { useDebounce } from 'use-debounce'
 import { ConfigurationFormProps, EVALUATION_SPECIFICATIONS } from './index'
-import {
-  ACCESSIBLE_OUTPUT_FORMATS,
-  ActualOutputConfiguration,
-  EvaluationMetric,
-  EvaluationType,
-  baseEvaluationConfiguration,
-} from '@latitude-data/core/constants'
 
-const MESSAGE_SELECTION_OPTIONS = baseEvaluationConfiguration.shape.actualOutput
-  .unwrap()
-  .shape.messageSelection.options.map((option) => ({
-    label: option.toUpperCase().split('_').join(' '),
-    value: option,
-  }))
+const MESSAGE_SELECTION_OPTIONS =
+  baseEvaluationConfiguration.shape.actualOutput.shape.messageSelection.options.map(
+    (option) => ({
+      label: option.toUpperCase().split('_').join(' '),
+      value: option,
+    }),
+  )
 
-const CONTENT_FILTER_OPTIONS = baseEvaluationConfiguration.shape.actualOutput
-  .unwrap()
-  .shape.contentFilter.unwrap()
-  .options.map((option) => ({
-    label: option.toUpperCase().split('_').join(' '),
-    value: option,
-  }))
+const CONTENT_FILTER_OPTIONS =
+  baseEvaluationConfiguration.shape.actualOutput.shape.contentFilter
+    .unwrap()
+    .options.map((option) => ({
+      label: option.toUpperCase().split('_').join(' '),
+      value: option,
+    }))
 
-const PARSING_FORMAT_OPTIONS = baseEvaluationConfiguration.shape.actualOutput
-  .unwrap()
-  .shape.parsingFormat.options.map((option) => ({
-    label: option.toUpperCase().split('_').join(' '),
-    value: option,
-  }))
+const PARSING_FORMAT_OPTIONS =
+  baseEvaluationConfiguration.shape.actualOutput.shape.parsingFormat.options.map(
+    (option) => ({
+      label: option.toUpperCase().split('_').join(' '),
+      value: option,
+    }),
+  )
 
 export function ConfigurationSimpleForm<
   T extends EvaluationType,
@@ -102,7 +104,7 @@ export function ConfigurationAdvancedForm<
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [formatIsAccessible])
 
-  const [testConfiguration] = useDebounce(configuration.actualOutput, 333)
+  const [testConfiguration] = useDebounce(configuration.actualOutput, 750)
   const [showTest, setShowTest] = useState(false)
 
   const typeSpecification = EVALUATION_SPECIFICATIONS[type]
@@ -150,7 +152,7 @@ export function ConfigurationAdvancedForm<
               }}
               onClick={(event) => {
                 event.preventDefault()
-                setShowTest(!showTest)
+                setShowTest(!showTest && !!configuration.actualOutput)
               }}
             >
               {showTest ? 'Close' : 'Test'}
@@ -209,6 +211,19 @@ export function ConfigurationAdvancedForm<
           label='Parsing format'
           description='How to parse the assistant messages'
           layout='horizontal'
+          tooltip={
+            formatIsAccessible ? (
+              <Text.H6 color='background'>
+                Use a field accessor to extract a specific field from the output
+                using dot notation.
+                <br />
+                <br />
+                - Access a field: arguments, arguments.options (nested)
+                <br />- Access a list: [0] (first), [-1] (last)
+                <br />- Combine both: [0].arguments.options[-1]
+              </Text.H6>
+            ) : undefined
+          }
         >
           <Select
             value={configuration.actualOutput?.parsingFormat ?? 'string'}
@@ -250,7 +265,7 @@ export function ConfigurationAdvancedForm<
           )}
         </FormFieldGroup>
       </FormFieldGroup>
-      {showTest && (
+      {showTest && !!configuration.actualOutput && (
         <div className='w-full flex flex-col gap-2 border-t-2 border-dashed border-border pt-4'>
           <ActualOutputTest configuration={testConfiguration} />
         </div>
@@ -262,7 +277,7 @@ export function ConfigurationAdvancedForm<
 function ActualOutputTest({
   configuration,
 }: {
-  configuration?: ActualOutputConfiguration
+  configuration: ActualOutputConfiguration
 }) {
   const { project } = useCurrentProject()
   const { commit } = useCurrentCommit()

apps/web/src/components/evaluations/ResultPanel.tsx

@@ -1,9 +1,13 @@
 'use client'
 
 import { DATASET_TABLE_PAGE_SIZE } from '$/app/(private)/datasets/_components/DatasetsTable'
-import { MetadataInfoTabs } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/MetadataInfoTabs'
 import { DocumentLogParameters } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/(withTabs)/logs/_components/DocumentLogs/DocumentLogInfo/Metadata'
+import { MetadataInfoTabs } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/MetadataInfoTabs'
 import { useCurrentDocument } from '$/app/providers/DocumentProvider'
+import {
+  useCurrentProject,
+  type IProjectContextType,
+} from '$/app/providers/ProjectProvider'
 import { MetadataItem } from '$/components/MetadataItem'
 import { useDatasetRole } from '$/hooks/useDatasetRoles'
 import { useStickyNested } from '$/hooks/useStickyNested'
@@ -12,7 +16,18 @@ import useDatasetRows from '$/stores/datasetRows'
 import useDatasetRowCount from '$/stores/datasetRows/count'
 import useDatasetRowPosition from '$/stores/datasetRows/position'
 import useDocumentLog from '$/stores/documentLogWithMetadata'
+import {
+  ACCESSIBLE_OUTPUT_FORMATS,
+  DocumentLog,
+  EvaluationMetric,
+  EvaluationResultV2,
+  EvaluationType,
+} from '@latitude-data/core/constants'
 import { buildPagination } from '@latitude-data/core/lib/pagination/buildPagination'
+import { Commit } from '@latitude-data/core/schema/models/types/Commit'
+import { Dataset } from '@latitude-data/core/schema/models/types/Dataset'
+import { DatasetRow } from '@latitude-data/core/schema/models/types/DatasetRow'
+import { DocumentVersion } from '@latitude-data/core/schema/models/types/DocumentVersion'
 import { Alert } from '@latitude-data/web-ui/atoms/Alert'
 import { Button } from '@latitude-data/web-ui/atoms/Button'
 import { Modal } from '@latitude-data/web-ui/atoms/Modal'
@@ -21,38 +36,12 @@ import { Text } from '@latitude-data/web-ui/atoms/Text'
 import { TextArea } from '@latitude-data/web-ui/atoms/TextArea'
 import { ClickToCopy } from '@latitude-data/web-ui/molecules/ClickToCopy'
 import { TableSkeleton } from '@latitude-data/web-ui/molecules/TableSkeleton'
-import {
-  useCurrentProject,
-  type IProjectContextType,
-} from '$/app/providers/ProjectProvider'
 import { format } from 'date-fns'
 import dynamic from 'next/dynamic'
 import Link from 'next/link'
 import { useEffect, useMemo, useRef, useState } from 'react'
 import { EVALUATION_SPECIFICATIONS, ResultPanelProps } from './index'
 import ResultBadge from './ResultBadge'
-import {
-  ACCESSIBLE_OUTPUT_FORMATS,
-  DocumentLog,
-  EvaluationMetric,
-  EvaluationResultV2,
-  EvaluationType,
-  baseEvaluationConfiguration,
-} from '@latitude-data/core/constants'
-import { Dataset } from '@latitude-data/core/schema/models/types/Dataset'
-
-import { Commit } from '@latitude-data/core/schema/models/types/Commit'
-import { DocumentVersion } from '@latitude-data/core/schema/models/types/DocumentVersion'
-import { DatasetRow } from '@latitude-data/core/schema/models/types/DatasetRow'
-const PARSING_FORMAT_LABELS = baseEvaluationConfiguration.shape.actualOutput
-  .unwrap()
-  .shape.parsingFormat.options.reduce(
-    (acc, option) => {
-      acc[option] = option.toUpperCase().split('_').join(' ')
-      return acc
-    },
-    {} as Record<string, string>,
-  )
 
 const DataGrid = dynamic(
   () =>
@@ -198,14 +187,13 @@ function ResultPanelMetadata<
               ) ? (
                 <Text.H6 color='foregroundMuted' noWrap ellipsis>
                   Parsed from{' '}
-                  {
-                    PARSING_FORMAT_LABELS[
-                      result.metadata!.configuration.actualOutput!.parsingFormat
-                    ]
-                  }
+                  {result
+                    .metadata!.configuration.actualOutput.parsingFormat.toUpperCase()
+                    .split('_')
+                    .join(' ')}
                   {!!result.metadata!.configuration.actualOutput
-                    ?.fieldAccessor &&
-                    ` using field '${result.metadata!.configuration.actualOutput!.fieldAccessor}'`}
+                    .fieldAccessor &&
+                    ` using field '${result.metadata!.configuration.actualOutput.fieldAccessor}'`}
                 </Text.H6>
               ) : (
                 <div />

apps/web/src/hooks/playgroundChat/useProviderEventHandler.ts

@@ -402,10 +402,7 @@ export function useProviderEventHandler({
           handleReasoningDelta(data)
           break
         case 'reasoning-start':
-          handleReasoningDelta({
-            type: 'reasoning-delta',
-            text: '',
-          })
+          handleReasoningDelta({ type: 'reasoning-delta', text: '' })
           break
         case 'file':
           handleFile(data)

apps/web/src/stores/evaluatedDocumentLogs.ts

@@ -1,12 +1,12 @@
 import useFetcher from '$/hooks/useFetcher'
 import { ROUTES } from '$/services/routes'
-import { useMemo, useState } from 'react'
-import useSWR, { SWRConfiguration } from 'swr'
 import {
   ActualOutputConfiguration,
   DocumentLogFilterOptions,
   EvaluatedDocumentLog,
 } from '@latitude-data/core/constants'
+import { useMemo, useState } from 'react'
+import useSWR, { SWRConfiguration } from 'swr'
 
 const EMPTY_ARRAY: [] = []
 export default function useEvaluatedDocumentLogs(
@@ -24,7 +24,7 @@ export default function useEvaluatedDocumentLogs(
     filterOptions: DocumentLogFilterOptions
     page: string | null | undefined
     pageSize: string | null
-    configuration?: ActualOutputConfiguration
+    configuration: ActualOutputConfiguration
     onFetched?: (logs: EvaluatedDocumentLog[]) => void
   },
   { fallbackData }: SWRConfiguration = {},
@@ -40,9 +40,7 @@ export default function useEvaluatedDocumentLogs(
             page: page ? Number(page) : undefined,
             pageSize: pageSize ? Number(pageSize) : undefined,
             filterOptions,
-            configuration: configuration
-              ? JSON.stringify(configuration)
-              : undefined,
+            configuration: JSON.stringify(configuration),
           })
       : undefined,
     {