diff --git a/.gitignore b/.gitignore index cf75843f..a7710165 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,13 @@ coverage.xml .tox/ .nox/ .pytest_cache + +# Test files and generated data +SeedsInstructions.json +seeds_housing.json +app/test_models.py +freeform_data_*.json + #old code app/frontend/ app/launch_streamlit.py diff --git a/app/client/src/api/Datasets/response.ts b/app/client/src/api/Datasets/response.ts index f3738346..3dc69cbd 100644 --- a/app/client/src/api/Datasets/response.ts +++ b/app/client/src/api/Datasets/response.ts @@ -17,6 +17,7 @@ export type DatasetResponse = { schema: string | null; custom_prompt: string; total_count: number; + completed_rows: number | null; num_questions: number; job_id: string; job_name: string; diff --git a/app/client/src/api/api.ts b/app/client/src/api/api.ts index b3439579..15b3de8d 100644 --- a/app/client/src/api/api.ts +++ b/app/client/src/api/api.ts @@ -13,12 +13,12 @@ import { const baseUrl = import.meta.env.VITE_AMP_URL; export const usefetchTopics = (useCase: string): UseFetchApiReturn => { - const url = `${baseUrl}/use-cases/${isEmpty(useCase) ? 'custom' : useCase}/topics`; + const url = isEmpty(useCase) ? '' : `${baseUrl}/use-cases/${useCase}/topics`; return useFetch(url); } export const useFetchExamples = (useCase: string): UseFetchApiReturn => { - const url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_examples`; + const url = isEmpty(useCase) ? '' : `${baseUrl}/${useCase}/gen_examples`; return useFetch(url); } @@ -27,21 +27,25 @@ export const useFetchModels = (): UseFetchApiReturn => { return useFetch(url); } -export const useFetchDefaultPrompt = (useCase: string, workflowType?: WorkerType): UseFetchApiReturn => { - let url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_prompt`; +export const useFetchDefaultPrompt = (useCase: string, workflowType?: string): UseFetchApiReturn => { + if (isEmpty(useCase)) { + return { data: null, loading: false, error: null }; + } + + let url = `${baseUrl}/${useCase}/gen_prompt`; if (workflowType && workflowType === 'freeform') { - url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_freeform_prompt`; + url = `${baseUrl}/${useCase}/gen_freeform_prompt`; } return useFetch(url); } -export const useFetchDefaultSchema = (): UseFetchApiReturn => { - const url = `${baseUrl}/sql_schema`; +export const useFetchDefaultSchema = (shouldFetch: boolean = true): UseFetchApiReturn => { + const url = shouldFetch ? `${baseUrl}/sql_schema` : ''; return useFetch(url); } -export const useFetchDefaultModelParams = (): UseFetchApiReturn => { - const url = `${baseUrl}/model/parameters`; +export const useFetchDefaultModelParams = (shouldFetch: boolean = true): UseFetchApiReturn => { + const url = shouldFetch ? `${baseUrl}/model/parameters` : ''; return useFetch(url); } diff --git a/app/client/src/api/hooks.ts b/app/client/src/api/hooks.ts index 81dc16a7..9070015b 100644 --- a/app/client/src/api/hooks.ts +++ b/app/client/src/api/hooks.ts @@ -1,15 +1,23 @@ import { useState, useMemo, useEffect } from 'react'; import { UseDeferredFetchApiReturn, UseFetchApiReturn } from './types'; - - +import { useQuery } from '@tanstack/react-query'; export function useFetch(url: string): UseFetchApiReturn { const [data, setData] = useState(null); - const [loading, setLoading] = useState(true); + const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const memoizedUrl = useMemo(() => url, [url]); useEffect(() => { + // Don't make API call if URL is empty (for regeneration scenarios) + if (!memoizedUrl || memoizedUrl.trim() === '') { + setData(null); + setLoading(false); + setError(null); + return; + } + + setLoading(true); const fetchData = async () => { try { const response = await fetch(memoizedUrl, { @@ -29,7 +37,12 @@ export function useFetch(url: string): UseFetchApiReturn { fetchData(); }, [memoizedUrl]); - return { data, loading, error }; + // Return false for loading when URL is empty + return { + data, + loading: (!memoizedUrl || memoizedUrl.trim() === '') ? false : loading, + error + }; } interface UseGetApiReturn { @@ -77,6 +90,7 @@ export function useGetApi(url: string): UseGetApiReturn { return { data, loading, error, triggerGet }; } + export function useDeferredFetch(url: string): UseDeferredFetchApiReturn { const [data, setData] = useState(null); const [loading, setLoading] = useState(true); @@ -192,3 +206,117 @@ export function useDeleteApi(url: string): UseDeleteApiReturn { return { data, loading, error, triggerDelete }; } +// Use case types and enums +export enum UseCaseId { + CODE_GENERATION = 'code_generation', + TEXT2SQL = 'text2sql', + CUSTOM = 'custom', + LENDING_DATA = 'lending_data', + CREDIT_CARD_DATA = 'credit_card_data', + TICKETING_DATASET = 'ticketing_dataset', +} + +export interface UseCase { + id: string; + name: string; +} + +export interface UseCasesResponse { + usecases: UseCase[]; +} + +const fetchUseCases = async (): Promise => { + const BASE_API_URL = import.meta.env.VITE_AMP_URL; + const response = await fetch(`${BASE_API_URL}/use-cases`); + if (!response.ok) { + throw new Error('Failed to fetch use cases'); + } + return response.json(); +}; + +export const useUseCases = () => { + return useQuery({ + queryKey: ['useCases'], + queryFn: fetchUseCases, + staleTime: 10 * 60 * 1000, // Cache for 10 minutes + retry: 3, // Retry 3 times on failure + retryDelay: (attemptIndex: number) => Math.min(1000 * 2 ** attemptIndex, 30000), // Exponential backoff + refetchOnWindowFocus: false, // Don't refetch when window gains focus + refetchOnMount: false, // Don't refetch on component mount if data exists + }); +}; + +export const useUseCaseMapping = () => { + const { data: useCasesData, isLoading, isError, error } = useUseCases(); + + // Create a lookup map for fast O(1) access + const useCaseMap = useMemo(() => { + if (!useCasesData?.usecases) return {}; + + return (useCasesData as UseCasesResponse).usecases.reduce((acc: Record, useCase: UseCase) => { + acc[useCase.id] = useCase.name; + return acc; + }, {} as Record); + }, [useCasesData]); + + // Helper function to get use case name with better fallback + const getUseCaseName = (id: string): string => { + if (isError) { + return id || 'N/A'; + } + + if (isLoading) { + return id || 'Loading...'; + } + + const name = useCaseMap[id]; + + // Log missing use cases in development + if (!name && id && typeof window !== 'undefined' && window.location.hostname === 'localhost') { + console.warn(`Missing use case mapping for: ${id}`); + } + + return name || id || 'N/A'; + }; + + // Get all use cases as array (useful for dropdowns) + const useCases = useMemo(() => { + return (useCasesData as UseCasesResponse)?.usecases || []; + }, [useCasesData]); + + return { + useCaseMap, + useCases, + getUseCaseName, + isLoading, + isError, + error + }; +}; + +// Hook to provide use case options for dropdowns and forms +export const useUseCaseOptions = () => { + const { useCases, isLoading, isError } = useUseCaseMapping(); + + // Transform use cases to option format used in dropdowns + const useCaseOptions = useMemo(() => { + return useCases.map((useCase: UseCase) => ({ + label: useCase.name, + value: useCase.id + })); + }, [useCases]); + + // Helper function to get use case type/name by id (replaces getUsecaseType) + const getUseCaseType = (id: string): string => { + const useCase = useCases.find((uc: UseCase) => uc.id === id); + return useCase?.name || id || 'N/A'; + }; + + return { + useCaseOptions, + getUseCaseType, + isLoading, + isError + }; +}; + diff --git a/app/client/src/components/Datasets/Datasets.tsx b/app/client/src/components/Datasets/Datasets.tsx index 049a2475..984bd6e9 100644 --- a/app/client/src/components/Datasets/Datasets.tsx +++ b/app/client/src/components/Datasets/Datasets.tsx @@ -12,7 +12,7 @@ import { Link } from "react-router-dom"; import { HuggingFaceIconUrl, Pages } from "../../types"; import { blue } from '@ant-design/colors'; import DateTime from "../DateTime/DateTime"; -import { TRANSLATIONS } from "../../constants"; +import { useUseCaseMapping } from "../../api/hooks"; import DeleteConfirmWarningModal from './DeleteConfirmModal'; import DatasetExportModal, { ExportResult } from '../Export/ExportModal'; @@ -22,6 +22,7 @@ const { Paragraph, Text } = Typography; export default function DatasetsComponent() { const datasetHistoryAPI = useGetDatasetHistory(); const deleteDatasetHistoryAPI = useDeleteDataset(); + const { getUseCaseName } = useUseCaseMapping(); const [toggleDatasetDetailModal, setToggleDatasetDetailModal] = React.useState(false); const [toggleDatasetExportModal, setToggleDatasetExportModal] = React.useState(false); const [exportResult, setExportResult] = React.useState(); @@ -65,7 +66,7 @@ export default function DatasetsComponent() { key: '3', title: 'Model', dataIndex: 'model_id', - render: (modelId) => {modelId} + render: (modelId: string) => {modelId} }, { key: '4', @@ -73,11 +74,19 @@ export default function DatasetsComponent() { dataIndex: 'num_questions', width: 150 }, + { + key: '4a', + title: 'Completed Rows', + dataIndex: 'completed_rows', + width: 120, + align: 'center', + render: (completed_rows: number | null) => <>{completed_rows != null ? completed_rows : 'N/A'} + }, { key: '5', title: 'Use Case', dataIndex: 'use_case', - render: (useCase) => {TRANSLATIONS[useCase]} + render: (useCase: string) => {getUseCaseName(useCase)} }, { key: '6', diff --git a/app/client/src/components/Evaluations/Evaluations.tsx b/app/client/src/components/Evaluations/Evaluations.tsx index 09dba710..135622e5 100644 --- a/app/client/src/components/Evaluations/Evaluations.tsx +++ b/app/client/src/components/Evaluations/Evaluations.tsx @@ -8,7 +8,7 @@ import DeleteIcon from '@mui/icons-material/Delete'; import { DownOutlined, FolderViewOutlined, ThunderboltOutlined } from '@ant-design/icons'; import { blue } from '@ant-design/colors'; -import { TRANSLATIONS } from "../../constants"; +import { useUseCaseMapping } from "../../api/hooks"; import DateTime from "../DateTime/DateTime"; import styled from "styled-components"; import { Pages } from "../../types"; @@ -23,6 +23,7 @@ const ModalButtonGroup = styled(Flex)` export default function Evaluations() { const evaluationsHistoryAPI = useGetEvaluationsHistory(); const deleteEvaluationHistoryAPI = useDeleteEvaluation(); + const { getUseCaseName } = useUseCaseMapping(); const [toggleEvaluationDetailModal, setToggleEvaluationDetailModal] = React.useState(false); const [evaluationDetail, setEvaluationDetail] = React.useState({} as EvaluationResponse); @@ -46,7 +47,7 @@ export default function Evaluations() { key: '4', title: 'Use Case', dataIndex: 'use_case', - render: (useCase) => {TRANSLATIONS[useCase]} + render: (useCase: string) => {getUseCaseName(useCase)} }, { key: '5', diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index bb7d42ea..57b8ba85 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -4,6 +4,7 @@ import isFunction from 'lodash/isFunction'; import { useEffect, useState } from 'react'; import { Flex, Form, Input, Select, Typography } from 'antd'; import styled from 'styled-components'; +import { useLocation } from 'react-router-dom'; import { File, WorkflowType } from './types'; import { useFetchModels } from '../../api/api'; import { MODEL_PROVIDER_LABELS } from './constants'; @@ -52,6 +53,7 @@ const Configure = () => { const formData = Form.useWatch((values) => values, form); const { setIsStepValid } = useWizardCtx(); const { data } = useFetchModels(); + const location = useLocation(); const [selectedFiles, setSelectedFiles] = useState( !isEmpty(form.getFieldValue('doc_paths')) ? form.getFieldValue('doc_paths') : []); @@ -73,12 +75,19 @@ const Configure = () => { validateForm() }, [form, formData]) - // keivan + // Only set default inference_type for completely new datasets useEffect(() => { - if (formData && formData?.inference_type === undefined) { + const isRegenerating = location.state?.data || location.state?.internalRedirect; + const existingInferenceType = form.getFieldValue('inference_type'); + + // Only set default if: + // 1. NOT regenerating an existing dataset + // 2. No existing inference_type value in form + // 3. formData watch shows undefined (initial state) + if (!isRegenerating && !existingInferenceType && formData && formData?.inference_type === undefined) { form.setFieldValue('inference_type', ModelProviders.CAII); } - }, [formData]); + }, [formData, location.state, form]); const labelCol = { span: 8 diff --git a/app/client/src/pages/DataGenerator/DataGenerator.tsx b/app/client/src/pages/DataGenerator/DataGenerator.tsx index bbf9b71f..753eca99 100644 --- a/app/client/src/pages/DataGenerator/DataGenerator.tsx +++ b/app/client/src/pages/DataGenerator/DataGenerator.tsx @@ -99,63 +99,99 @@ const DataGenerator = () => { const [current, setCurrent] = useState(0); const [maxStep, setMaxStep] = useState(0); const [isStepValid, setIsStepValid] = useState(false); + const [formInitialValues, setFormInitialValues] = useState(null); // Data passed from listing table to prepopulate form const location = useLocation(); const { generate_file_name } = useParams(); - const initialData = location?.state?.data; + let initialData = location?.state?.data; const mutation = useMutation({ mutationFn: fetchDatasetDetails }); - - useEffect(() => { - if (generate_file_name && !mutation.data) { - mutation.mutate(generate_file_name); - } - if (mutation.data && mutation?.data?.dataset) { - form.setFieldsValue({ - ...initialData, - ...(mutation?.data?.dataset as any) - }); - } - - }, [generate_file_name]); - - + // Process initial data for regeneration if (initialData?.technique) { - initialData.workflow_type = initialData?.technique === 'sft' ? - WorkflowType.SUPERVISED_FINE_TUNING : - initialData?.technique === 'freeform' ? WorkflowType.FREE_FORM_DATA_GENERATION : - WorkflowType.CUSTOM_DATA_GENERATION; + initialData = { + ...initialData, + workflow_type: initialData.technique // Use technique value directly as it matches WORKFLOW_OPTIONS + }; } - if (Array.isArray(initialData?.doc_paths) && !isEmpty(initialData?.doc_paths) ) { - initialData.doc_paths = initialData?.doc_paths.map((path: string) => ({ - value: path, - label: path - })); - + + if (Array.isArray(initialData?.doc_paths) && !isEmpty(initialData?.doc_paths)) { + initialData = { + ...initialData, + doc_paths: initialData.doc_paths.map((path: string) => ({ + value: path, + label: path + })) + }; } - // if (datasetDetailsReq && datasetDetailsReq.data && - // !isEmpty(datasetDetailsReq?.data?.generate_file_name)) { - // initialData.example_path = initialData?.example_path; - // } - - if (Array.isArray(initialData?.input_paths) && !isEmpty(initialData?.input_paths) ) { - initialData.doc_paths = initialData?.input_paths.map((path: string) => ({ - value: path, - label: path - })); + if (Array.isArray(initialData?.input_paths) && !isEmpty(initialData?.input_paths)) { + initialData = { + ...initialData, + doc_paths: initialData.input_paths.map((path: string) => ({ + value: path, + label: path + })) + }; } + if (isString(initialData?.doc_paths)) { - initialData.doc_paths = []; + initialData = { + ...initialData, + doc_paths: [] + }; } + const [form] = Form.useForm(); - const formData = useRef(initialData || { num_questions: 20, topics: [] }); + // Set initial form values based on available data + useEffect(() => { + if (initialData) { + // We have data from location.state (actions menu regeneration) + setFormInitialValues(initialData); + } else if (!generate_file_name) { + // New dataset creation + setFormInitialValues({ num_questions: 20, topics: [] }); + } + }, [initialData, generate_file_name]); - const [form] = Form.useForm(); + useEffect(() => { + if (generate_file_name && !mutation.data) { + mutation.mutate(generate_file_name); + } + if (mutation.data && mutation?.data?.dataset) { + const apiData = mutation.data.dataset as any; + + // Map technique from API to workflow_type for UI + let processedApiData = { ...apiData }; + if (apiData.technique) { + processedApiData.workflow_type = apiData.technique; // Use technique value directly as it matches WORKFLOW_OPTIONS + } + + // Process doc_paths for API data + if (Array.isArray(apiData.doc_paths) && !isEmpty(apiData.doc_paths)) { + processedApiData.doc_paths = apiData.doc_paths.map((path: string) => ({ + value: path, + label: path + })); + } + + if (Array.isArray(apiData.input_paths) && !isEmpty(apiData.input_paths)) { + processedApiData.doc_paths = apiData.input_paths.map((path: string) => ({ + value: path, + label: path + })); + } + + const finalFormValues = { ...initialData, ...processedApiData }; + + // Update both the form values and the initial values state + setFormInitialValues(finalFormValues); + form.setFieldsValue(finalFormValues); + } + }, [generate_file_name, mutation.data]); const onStepChange = (value: number) => { setCurrent(value); @@ -180,15 +216,17 @@ const DataGenerator = () => { items={steps.map((step, i) => ({ title: step.title, key: step.key, disabled: maxStep < i }))} /> -
{}} - > - {steps[current].content} -
+ {formInitialValues && ( +
{}} + > + {steps[current].content} +
+ )}
diff --git a/app/client/src/pages/DataGenerator/Examples.tsx b/app/client/src/pages/DataGenerator/Examples.tsx index 61d71acf..43a8da1e 100644 --- a/app/client/src/pages/DataGenerator/Examples.tsx +++ b/app/client/src/pages/DataGenerator/Examples.tsx @@ -1,21 +1,33 @@ import first from 'lodash/first'; import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; -import React, { useEffect } from 'react'; -import { Button, Form, Modal, Space, Table, Tooltip, Typography, Flex, Input, Empty } from 'antd'; -import { CloudUploadOutlined, DeleteOutlined, EditOutlined } from '@ant-design/icons'; +import React, { useEffect, useMemo, useCallback, useState } from 'react'; +import { Button, Form, Modal, Space, Typography, Flex, Input, Empty, Spin } from 'antd'; +import { CloudUploadOutlined } from '@ant-design/icons'; import styled from 'styled-components'; import { useMutation } from "@tanstack/react-query"; -import { useFetchExamples } from '../../api/api'; +import { useLocation } from 'react-router-dom'; +import { AgGridReact } from 'ag-grid-react'; +import { themeMaterial, ModuleRegistry, ClientSideRowModelModule, ValidationModule, TextFilterModule, NumberFilterModule, DateFilterModule, type ColDef, type GetRowIdFunc, type GetRowIdParams, type ICellRendererParams } from 'ag-grid-community'; +import toString from 'lodash/toString'; + import TooltipIcon from '../../components/TooltipIcon'; import PCModalContent from './PCModalContent'; import { ExampleType, File, QuestionSolution, WorkflowType } from './types'; import FileSelectorButton from './FileSelectorButton'; import { fetchFileContent, getExampleType, useGetExamplesByUseCase } from './hooks'; -import { useState } from 'react'; import FreeFormExampleTable from './FreeFormExampleTable'; +// Register AG Grid modules +ModuleRegistry.registerModules([ + TextFilterModule, + NumberFilterModule, + DateFilterModule, + ClientSideRowModelModule, + ValidationModule +]); + const { Title, Text } = Typography; const Container = styled.div` padding-bottom: 10px @@ -27,14 +39,6 @@ const Header = styled(Flex)` const StyledTitle = styled(Title)` margin: 0; ` -const ModalButtonGroup = styled(Flex)` - margin-top: 15px !important; -` -const StyledTable = styled(Table)` - .ant-table-row { - cursor: pointer; - } -` const StyledContainer = styled.div` margin-bottom: 24px; @@ -43,158 +47,206 @@ const StyledContainer = styled.div` svg { font-size: 48px; } +`; +const LoadingContainer = styled.div` + height: 400px; + display: flex; + align-items: center; + justify-content: center; + flex-direction: column; + gap: 16px; `; -const MAX_EXAMPLES = 5; +// Simple cell renderer without click interactions +const TextCellRenderer = (params: ICellRendererParams) => { + const { value } = params; + if (!value) return ''; + + return ( +
+ {value} +
+ ); +}; + +// Unified AG Grid Table Component for all templates - READ-ONLY +const UnifiedExampleTable: React.FC<{ data: QuestionSolution[], loading?: boolean }> = ({ data, loading = false }) => { + const [colDefs, setColDefs] = useState([]); + const [rowData, setRowData] = useState([]); + + useEffect(() => { + if (!isEmpty(data)) { + const columnDefs: ColDef[] = [ + { + field: 'question', + headerName: 'Prompts', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 300, + cellRenderer: TextCellRenderer, + wrapText: true, + autoHeight: false, + }, + { + field: 'solution', + headerName: 'Completions', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 300, + cellRenderer: TextCellRenderer, + wrapText: true, + autoHeight: false, + } + ]; + setColDefs(columnDefs); + setRowData(data); + } + }, [data]); + + const defaultColDef: ColDef = useMemo( + () => ({ + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 250, + }), + [] + ); + + let index = 0; + const getRowId = useCallback( + ({ data: rowData }: GetRowIdParams) => { + index++; + return toString(index); + }, + [] + ); + + if (loading) { + return ( + + + Loading examples... + + ); + } + if (isEmpty(data)) { + return ( +
+ +
+ ); + } + return ( +
+ +
+ ); +}; const Examples: React.FC = () => { const form = Form.useFormInstance(); + const location = useLocation(); const [exampleType, setExampleType] = useState(ExampleType.PROMPT_COMPLETION); const mutation = useMutation({ mutationFn: fetchFileContent }); - const values = form.getFieldsValue(true) + + // Get ALL form values - this works consistently during regeneration + const allFormValues = form.getFieldsValue(true); + const { examples = [] } = allFormValues; + + // Check if this is a regeneration scenario + const isRegenerating = location.state?.data || location.state?.internalRedirect; + const useCase = form.getFieldValue('use_case'); + const workflowType = form.getFieldValue('workflow_type'); + + // CRITICAL FIX: Determine example type immediately when examples are available + // This prevents race conditions and eliminates back-and-forth loading issues + const currentExampleType = useMemo(() => { + // Priority 1: If workflow is freeform, always use FREE_FORM + if (workflowType === 'freeform') { + return ExampleType.FREE_FORM; + } + + // Priority 2: If we have examples data, determine type from the data structure + if (!isEmpty(examples)) { + return getExampleType(examples) as ExampleType; + } + + // Priority 3: If we have mutation data (file upload), it's always FREE_FORM + if (!isEmpty(mutation.data)) { + return ExampleType.FREE_FORM; + } + + // Priority 4: Default to PROMPT_COMPLETION for 2-column templates + return ExampleType.PROMPT_COMPLETION; + }, [workflowType, examples, mutation.data]); useEffect(() => { const example_path = form.getFieldValue('example_path'); - if (!isEmpty(example_path)) { + // Only try to load from example_path if we're not regenerating and have a path + if (!isRegenerating && !isEmpty(example_path)) { mutation.mutate({ path: example_path }); } - - if (form.getFieldValue('workflow_type') === 'freeform') { - setExampleType(ExampleType.FREE_FORM); - } - - - - }, [form.getFieldValue('example_path'), form.getFieldValue('workflow_type')]); + }, [form.getFieldValue('example_path'), isRegenerating]); useEffect(() => { - if (!isEmpty(mutation.data)) { + // Only set examples from mutation data if we're not regenerating + if (!isRegenerating && !isEmpty(mutation.data)) { form.setFieldValue('examples', mutation.data); } - }, [mutation.data]); - - const columns = [ - { - title: 'Prompts', - dataIndex: 'question', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => {record.question} - }, - { - title: 'Completions', - dataIndex: 'solution', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => {record.solution} - }, - { - title: 'Actions', - key: 'actions', - width: 130, - render: (_text: QuestionSolution, record: QuestionSolution, index: number) => { - const { question, solution } = record; - const editRow = (data: QuestionSolution) => { - const updatedExamples = [...form.getFieldValue('examples')]; - updatedExamples.splice(index, 1, data); - form.setFieldValue('examples', updatedExamples); - Modal.destroyAll() - } - const deleteRow = () => { - const updatedExamples = [...form.getFieldValue('examples')]; - updatedExamples.splice(index, 1); - form.setFieldValue('examples', updatedExamples); - } - return ( - - - - - ), - maskClosable: true, - width: 1000 - }) - }} - /> - - ) - } - }, - ]; - const dataSource = Form.useWatch('examples', form); - const { examples, exmpleFormat, isLoading: examplesLoading } = - useGetExamplesByUseCase(form.getFieldValue('use_case')); + }, [mutation.data, isRegenerating]); - // update examples - if (!dataSource && examples) { - form.setFieldValue('examples', examples) - } + // CRITICAL FIX: Don't make API call at all during regeneration or when we already have examples + const shouldFetchFromAPI = !isRegenerating && isEmpty(examples) && !isEmpty(useCase); + const { examples: apiExamples, exmpleFormat, isLoading: examplesLoading } = + useGetExamplesByUseCase(shouldFetchFromAPI ? useCase : ''); + + // Only update examples from API if we're not regenerating and don't have existing examples useEffect(() => { - if (!isEmpty(examples) && !isEmpty(exmpleFormat)) { - setExampleType(exmpleFormat as ExampleType); - form.setFieldValue('examples', examples || []); + if (!isRegenerating && isEmpty(examples) && apiExamples) { + form.setFieldValue('examples', apiExamples) } - }, [examples, exmpleFormat]); - - const rowLimitReached = form.getFieldValue('examples')?.length === MAX_EXAMPLES; - const workflowType = form.getFieldValue('workflow_type'); + }, [apiExamples, examples, isRegenerating]); + + // REMOVED: The problematic useEffect that caused race conditions + // The exampleType is now determined synchronously via useMemo above const onAddFiles = (files: File[]) => { if (!isEmpty (files)) { @@ -207,7 +259,6 @@ const Examples: React.FC = () => { ...values, example_path: get(file, '_path') }); - setExampleType(ExampleType.FREE_FORM); } } @@ -215,21 +266,25 @@ const Examples: React.FC = () => { span: 10 }; + // FIXED: Show loading when we should fetch from API AND the API is actually loading + // This ensures the spinner shows up when the API call is in progress + const shouldShowLoading = shouldFetchFromAPI && examplesLoading; + return (
<>{'Examples'} - + - {workflowType === WorkflowType.FREE_FORM_DATA_GENERATION && + {workflowType === 'freeform' && <> { } - - {exampleType !== ExampleType.FREE_FORM && - - - - ), - maskClosable: true, - }) - }} - > - {'Restore Defaults'} - } - - {exampleType !== ExampleType.FREE_FORM && - - - }
- {exampleType === ExampleType.FREE_FORM && !isEmpty(mutation.data) && - } - {exampleType === ExampleType.FREE_FORM && form.getFieldValue('use_case') === 'lending_data' && - } - {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && !isEmpty(values.examples) && - } - {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && isEmpty(values.examples) && - - - - } - imageStyle={{ - height: 60, - marginBottom: 24 - }} - description={ - <> -

- Upload a JSON file containing examples -

-

- {'Examples should be in the format of a JSON array containing array of key & value pairs. The key should be the column name and the value should be the cell value.'} -

- - } + {currentExampleType === ExampleType.FREE_FORM ? ( + workflowType === 'freeform' && isEmpty(examples) && isEmpty(mutation.data) && !shouldShowLoading ? ( + + + + } + imageStyle={{ + height: 60, + marginBottom: 24 + }} + description={ + <> +

+ Upload a JSON file containing examples +

+

+ {'Examples should be in the format of a JSON array containing array of key & value pairs. The key should be the column name and the value should be the cell value.'} +

+ + } + > + + ) : ( + + ) + ) : ( + - - } - {exampleType !== ExampleType.FREE_FORM && - - ({ - onClick: () => Modal.info({ - title: 'View Details', - content: , - icon: undefined, - maskClosable: true, - width: 1000 - }) - })} - rowClassName={() => 'hover-pointer'} - rowKey={(_record, index) => `examples-table-${index}`} - /> - } + + + )} ) diff --git a/app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx b/app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx index c93bceba..4d2fd46d 100644 --- a/app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx +++ b/app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx @@ -4,6 +4,8 @@ import first from 'lodash/first'; import toString from 'lodash/toString'; import React, { FunctionComponent, useState, useMemo, useCallback, useEffect } from 'react'; import { AgGridReact } from 'ag-grid-react'; +import { Spin, Empty, Typography } from 'antd'; +import styled from 'styled-components'; // // Register all Community features // // ModuleRegistry.registerModules([AllCommunityModule]); @@ -41,27 +43,42 @@ ModuleRegistry.registerModules([ ValidationModule ]); +const { Text } = Typography; + +const LoadingContainer = styled.div` + height: 600px; + display: flex; + align-items: center; + justify-content: center; + flex-direction: column; + gap: 16px; +`; + interface Props { data: Record[]; + loading?: boolean; } -const FreeFormExampleTable: FunctionComponent = ({ data }) => { - const [colDefs, setColDefs] = useState([]); - const [rowData, setRowData] = useState([]); +const FreeFormExampleTable: FunctionComponent = ({ data, loading = false }) => { + const [colDefs, setColDefs] = useState([]); + const [rowData, setRowData] = useState[]>([]); useEffect(() => { if (!isEmpty(data)) { - const columnNames = Object.keys(first(data)); - const columnDefs = columnNames.map((colName) => ({ - field: colName, - headerName: colName, - width: 250, - filter: true, - sortable: true, - resizable: true - })); - setColDefs(columnDefs); - setRowData(data); + const firstRow = first(data); + if (firstRow) { + const columnNames = Object.keys(firstRow); + const columnDefs = columnNames.map((colName) => ({ + field: colName, + headerName: colName, + width: 250, + filter: true, + sortable: true, + resizable: true + })); + setColDefs(columnDefs); + setRowData(data); + } } } , [data]); @@ -72,8 +89,7 @@ const FreeFormExampleTable: FunctionComponent = ({ data }) => { filter: true, enableRowGroup: true, enableValue: true, - - editable: true, + editable: false, // Make it non-editable for consistency minWidth: 170 }), [] @@ -101,13 +117,31 @@ const FreeFormExampleTable: FunctionComponent = ({ data }) => { [] ); + // Show loading state + if (loading) { + return ( + + + Loading examples... + + ); + } + + // Show empty state if no data + if (isEmpty(data)) { + return ( +
+ +
+ ); + } return ( <>
= ({ data }) => { getRowId={getRowId} defaultColDef={defaultColDef} statusBar={statusBar} + suppressRowHoverHighlight={true} // Remove hover effects for consistency + suppressCellFocus={true} />
diff --git a/app/client/src/pages/DataGenerator/Parameters.tsx b/app/client/src/pages/DataGenerator/Parameters.tsx index 65db49f9..8e554292 100644 --- a/app/client/src/pages/DataGenerator/Parameters.tsx +++ b/app/client/src/pages/DataGenerator/Parameters.tsx @@ -3,6 +3,7 @@ import { useEffect, useRef, useState } from 'react'; import { Col, Divider, Form, InputNumber, Row, Slider, Typography } from 'antd'; import { merge } from 'lodash'; import styled from 'styled-components'; +import { useLocation } from 'react-router-dom'; import { useFetchDefaultModelParams } from '../../api/api'; import { LABELS } from '../../constants'; @@ -27,6 +28,10 @@ const ParamLabel = styled(Typography)` const Parameters = () => { const form = Form.useFormInstance() + const location = useLocation(); + + // Check if this is a regeneration scenario + const isRegenerating = location.state?.data || location.state?.internalRedirect; const MODEL_PARAM_DEFAULTS = useRef({ temperature: { @@ -58,7 +63,8 @@ const Parameters = () => { const formData = form.getFieldsValue(true); const [values, setValues] = useState(formData?.model_parameters); - const { data: defaultParams } = useFetchDefaultModelParams(); + // Only fetch default params if we're NOT regenerating + const { data: defaultParams } = useFetchDefaultModelParams(!isRegenerating); useEffect(() => { if (!isEmpty(formData?.model_parameters)) { @@ -67,7 +73,8 @@ const Parameters = () => { }, [formData?.model_parameters]); useEffect(() => { - if (defaultParams && !formData.model_parameters) { + // Only set defaults for new datasets, not during regeneration + if (!isRegenerating && defaultParams && !formData.model_parameters) { // Set ref to be use to define min/max for each formItem MODEL_PARAM_DEFAULTS.current = merge(MODEL_PARAM_DEFAULTS.current, defaultParams.parameters); // Create the data structure to set the default value for each form item. @@ -77,8 +84,7 @@ const Parameters = () => { setValues(defaultValues) form.setFieldValue('model_parameters', defaultValues) } - - }, [defaultParams]); + }, [defaultParams, isRegenerating]); // Update both InputNumber and Slider together const handleValueChange = (field: string, value: string | number | null) => { diff --git a/app/client/src/pages/DataGenerator/Prompt.tsx b/app/client/src/pages/DataGenerator/Prompt.tsx index 45f8b798..fba20060 100644 --- a/app/client/src/pages/DataGenerator/Prompt.tsx +++ b/app/client/src/pages/DataGenerator/Prompt.tsx @@ -5,6 +5,7 @@ import { PlusOutlined } from '@ant-design/icons'; import { Alert, Button, Col, Divider, Flex, Form, Input, InputNumber, Modal, notification, Row, Select, Space, Tooltip, Typography } from 'antd'; import type { InputRef } from 'antd'; import styled from 'styled-components'; +import { useLocation } from 'react-router-dom'; import Parameters from './Parameters'; import TooltipIcon from '../../components/TooltipIcon'; @@ -20,6 +21,7 @@ import FileSelectorButton from './FileSelectorButton'; import { useMutation } from '@tanstack/react-query'; import first from 'lodash/first'; import ResetIcon from './ResetIcon'; +import { File } from './types'; const { Title } = Typography; @@ -76,12 +78,17 @@ const SeedsFormItem = styled(StyledFormItem)` const Prompt = () => { const form = Form.useFormInstance(); + const location = useLocation(); const selectedTopics = Form.useWatch('topics'); const numQuestions = Form.useWatch('num_questions'); const datasetSize = Form.useWatch('num_questions'); const [items, setItems] = useState([]); const [customTopic, setCustomTopic] = useState(''); + // Check if this is a regeneration scenario + const isRegenerating = location.state?.data || location.state?.internalRedirect; + const existingTopics = form.getFieldValue('topics'); + const customTopicRef = useRef(null); const defaultPromptRef = useRef(null); const defaultSchemaRef = useRef(null); @@ -96,11 +103,11 @@ const Prompt = () => { const output_key = form.getFieldValue('output_key'); const caii_endpoint = form.getFieldValue('caii_endpoint'); - const { data: defaultPrompt, loading: promptsLoading } = useFetchDefaultPrompt(useCase, workflow_type); - - // Page Bootstrap requests and useEffect - const { data: defaultTopics, loading: topicsLoading } = usefetchTopics(useCase); - const { data: defaultSchema, loading: schemaLoading } = useFetchDefaultSchema(); + // Only fetch defaults if we're NOT regenerating + const { data: defaultPrompt, loading: promptsLoading } = useFetchDefaultPrompt(!isRegenerating ? useCase : '', workflow_type); + const { data: defaultTopics, loading: topicsLoading } = usefetchTopics(!isRegenerating ? useCase : ''); + const { data: defaultSchema, loading: schemaLoading } = useFetchDefaultSchema(!isRegenerating); + const { data: dataset_size, isLoading: datasetSizeLoading, isError, error } = useDatasetSize( workflow_type, doc_paths, @@ -120,7 +127,6 @@ const Prompt = () => { } }, [mutation.data]); - useEffect(() => { if (isError) { notification.error({ @@ -128,11 +134,15 @@ const Prompt = () => { description: get(error, 'error'), }); } - }, [error, isError]); useEffect(() => { - if (defaultTopics) { + // For regeneration: use existing topics if available + if (isRegenerating && !isEmpty(existingTopics)) { + setItems(existingTopics); + } + // For new datasets: use API topics + else if (!isRegenerating && defaultTopics) { // customTopics is a client-side only fieldValue that persists custom topics added // when the user switches between wizard steps const customTopics = form.getFieldValue('customTopics') @@ -145,7 +155,9 @@ const Prompt = () => { form.setFieldValue('topics', []) } } - if (defaultPrompt) { + + // Handle prompts - for regeneration, preserve existing; for new, use defaults + if (!isRegenerating && defaultPrompt) { defaultPromptRef.current = defaultPrompt; if (form.getFieldValue('custom_prompt') === undefined) { form.setFieldValue('custom_prompt', defaultPrompt) @@ -155,18 +167,21 @@ const Prompt = () => { form.setFieldValue('custom_prompt', defaultPrompt) } } - if (defaultSchema) { + + // Handle schema - only for new datasets + if (!isRegenerating && defaultSchema) { defaultSchemaRef.current = defaultSchema; if (form.getFieldValue('schema') === undefined) { form.setFieldValue('schema', defaultSchema) } } + if (dataset_size) { if (form.getFieldValue('num_questions') !== dataset_size) { form.setFieldValue('num_questions', dataset_size) } } - }, [defaultPromptRef, defaultSchema, defaultSchemaRef, defaultTopics, dataset_size, form, setItems]); + }, [defaultPromptRef, defaultSchema, defaultSchemaRef, defaultTopics, dataset_size, form, setItems, isRegenerating, existingTopics]); const { setIsStepValid } = useWizardCtx(); useEffect(() => { @@ -176,7 +191,7 @@ const Prompt = () => { isStepValid = true; } setIsStepValid(isStepValid) - } else if(!isEmpty(doc_paths) && workflow_type === WorkflowType.SUPERVISED_FINE_TUNING) { + } else if(!isEmpty(doc_paths) && workflow_type === 'sft') { setIsStepValid(isNumber(datasetSize) && datasetSize > 0); } else { setIsStepValid(true); @@ -244,8 +259,7 @@ const Prompt = () => { - {(form.getFieldValue('use_case') === Usecases.CUSTOM.toLowerCase() || - workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) && + {form.getFieldValue('use_case') === 'custom' && {
- {((workflow_type === WorkflowType.CUSTOM_DATA_GENERATION && !isEmpty(doc_paths)) || - (workflow_type === WorkflowType.SUPERVISED_FINE_TUNING && !isEmpty(doc_paths))) && + {((workflow_type === 'custom_workflow' && !isEmpty(doc_paths)) || + (workflow_type === 'sft' && !isEmpty(doc_paths))) && { } rules={[ - { required: workflow_type === WorkflowType.SUPERVISED_FINE_TUNING, message: `Please select a workflow.` } + { required: workflow_type === 'sft', message: `Please select a workflow.` } ]} labelCol={{ span: 24 }} wrapperCol={{ span: 24 }} @@ -315,12 +329,12 @@ const Prompt = () => { > - + } - {isEmpty(doc_paths) && (workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || - workflow_type === WorkflowType.CUSTOM_DATA_GENERATION || - workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) && + {isEmpty(doc_paths) && (workflow_type === 'sft' || + workflow_type === 'custom_workflow' || + workflow_type === 'freeform') && {mutation.isError && = { @@ -23,12 +38,6 @@ const MarkdownWrapper = styled.div` padding: 4px 11px; `; -const StyledTable = styled(Table)` - .ant-table-row { - cursor: pointer; - } -` - const StyledTextArea = styled(Input.TextArea)` color: #575757 !important; background: #fafafa !important; @@ -37,6 +46,113 @@ const StyledTextArea = styled(Input.TextArea)` } `; +// Improved cell renderer with better text handling +const TextCellRenderer = (params: ICellRendererParams) => { + const { value } = params; + if (!value) return ''; + + return ( +
+ {value} +
+ ); +}; + +const SummaryExampleTable: React.FC<{ data: QuestionSolution[] }> = ({ data }) => { + const [colDefs, setColDefs] = useState([]); + const [rowData, setRowData] = useState([]); + + useEffect(() => { + if (!isEmpty(data)) { + const columnDefs: ColDef[] = [ + { + field: 'question', + headerName: 'Prompts', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 300, + cellRenderer: TextCellRenderer, + wrapText: true, + autoHeight: false, + }, + { + field: 'solution', + headerName: 'Completions', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 300, + cellRenderer: TextCellRenderer, + wrapText: true, + autoHeight: false, + } + ]; + setColDefs(columnDefs); + setRowData(data); + } + }, [data]); + + const defaultColDef: ColDef = useMemo( + () => ({ + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 250, + }), + [] + ); + + let index = 0; + const getRowId = useCallback( + ({ data: rowData }: GetRowIdParams) => { + index++; + return toString(index); + }, + [] + ); + + if (isEmpty(data)) { + return ( +
+ +
+ ); + } + + return ( +
+ +
+ ); +}; + const Summary= () => { const form = Form.useFormInstance() const { @@ -63,72 +179,21 @@ const Summary= () => { { label: 'Data Count', children: num_questions }, { label: 'Total Dataset Size', children: topics === null ? num_questions : num_questions * topics.length }, ]; - const exampleCols = [ - { - title: 'Prompts', - dataIndex: 'prompts', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.question} - }, - { - title: 'Completions', - dataIndex: 'completions', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.solution} - }, - ]; return ( - +
- {'Settings'} - {'Configuration'} +
{'Prompt'} - - {/* - - */} +
-
- {'Parameters'} - { - return { - key: `${param}-${i}`, - label: MODEL_PARAMETER_LABELS[param as ModelParameters], - children: model_parameters[param] - } - })} - size='small' - style={{ maxWidth: 400}} - /> -
- {isEmpty(topics) && -
- {'Seed Instructions'} - ({item})} - locale={{ - emptyText: ( - - {'No seed instructions were selected'} - - ) - }} - /> -
} {(schema && use_case === Usecases.TEXT2SQL) && (
{'DB Schema'} @@ -142,25 +207,10 @@ const Summary= () => { {'Examples'} {workflow_type === 'freeform' ? : - ({ - onClick: () => Modal.info({ - title: 'View Details', - content: , - icon: undefined, - maskClosable: true, - width: 1000 - }) - })} - rowKey={(_record, index) => `summary-examples-table-${index}`} - />} + }
} ) -} +}; export default Summary; \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx index bb38a351..d5a953cb 100644 --- a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx +++ b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx @@ -1,18 +1,24 @@ import { Form, Select } from "antd"; import { FunctionComponent, useEffect, useState } from "react"; +import { useLocation } from "react-router-dom"; import { useGetUseCases } from "./hooks"; import { UseCase } from "../../types"; import get from "lodash/get"; interface Props {} - const UseCaseSelector: FunctionComponent = () => { const [useCases, setUseCases] = useState([]); - const useCasesReq = useGetUseCases(); + const location = useLocation(); + + // Check if this is a regeneration scenario + const isRegenerating = location.state?.data || location.state?.internalRedirect; + + // Only fetch use cases if we're NOT regenerating + const useCasesReq = useGetUseCases(isRegenerating); useEffect(() => { - if (useCasesReq.data) { + if (!isRegenerating && useCasesReq.data) { let _useCases = get(useCasesReq, 'data.usecases', []); _useCases = _useCases.map((useCase: any) => ({ ...useCase, @@ -21,8 +27,7 @@ const UseCaseSelector: FunctionComponent = () => { })); setUseCases(_useCases); } - }, [useCasesReq.data]); - + }, [useCasesReq.data, isRegenerating]); return ( { +export const useGetUseCases = (shouldSkip: boolean = false) => { const { data, isLoading, isError, error, isFetching } = useQuery( { queryKey: ['useCases'], queryFn: () => fetchUseCases(), refetchOnWindowFocus: false, + enabled: !shouldSkip, // Only run query if not skipped } ); return { data, - isLoading: isLoading || isFetching, + isLoading: shouldSkip ? false : (isLoading || isFetching), // Return false when disabled isError, error }; @@ -282,9 +283,10 @@ export const fetchExamplesByUseCase = async (use_case: string) => { export const useGetExamplesByUseCase = (use_case: string) => { const { data, isLoading, isError, error, isFetching } = useQuery( { - queryKey: ['fetchUseCaseTopics', fetchExamplesByUseCase], + queryKey: ['fetchUseCaseTopics', fetchExamplesByUseCase, use_case], queryFn: () => fetchExamplesByUseCase(use_case), refetchOnWindowFocus: false, + enabled: !isEmpty(use_case), // Only run query if use_case is not empty } ); @@ -305,7 +307,7 @@ export const useGetExamplesByUseCase = (use_case: string) => { return { data, - isLoading: isLoading || isFetching, + isLoading: isLoading || isFetching, // Always return actual loading state isError, error, examples, diff --git a/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx b/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx index b16ec0ca..cc5c7b13 100644 --- a/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx +++ b/app/client/src/pages/DatasetDetails/ConfigurationTab.tsx @@ -1,59 +1,46 @@ import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; -import React from 'react'; +import React, { useEffect, useMemo, useCallback, useState } from 'react'; import { Dataset } from '../Evaluator/types'; -import { Col, Flex, Modal, Row, Space, Table, Tag, Typography } from 'antd'; -import ExampleModal from './ExampleModal'; -import { QuestionSolution } from '../DataGenerator/types'; +import { Col, Flex, Modal, Row } from 'antd'; import styled from 'styled-components'; -import FreeFormExampleTable from '../DataGenerator/FreeFormExampleTable'; +import { AgGridReact } from 'ag-grid-react'; +import { themeMaterial, ModuleRegistry, ClientSideRowModelModule, ValidationModule, TextFilterModule, NumberFilterModule, DateFilterModule, type ColDef, type GetRowIdFunc, type GetRowIdParams } from 'ag-grid-community'; +import toString from 'lodash/toString'; -const { Text } = Typography; +import { QuestionSolution } from '../DataGenerator/types'; +import FreeFormExampleTable from '../DataGenerator/FreeFormExampleTable'; +import ExampleModal from './ExampleModal'; +import { ICellRendererParams } from 'ag-grid-community'; +import { Empty } from 'antd'; +import PCModalContent from '../DataGenerator/PCModalContent'; + +// Register AG Grid modules +ModuleRegistry.registerModules([ + TextFilterModule, + NumberFilterModule, + DateFilterModule, + ClientSideRowModelModule, + ValidationModule +]); interface Props { dataset: Dataset; } -const StyledTable = styled(Table)` - font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; - color: #5a656d; - .ant-table-thead > tr > th { - color: #5a656d; - border-bottom: 1px solid #eaebec; - font-weight: 500; - text-align: left; - // background: #ffffff; - border-bottom: 1px solid #eaebec; - transition: background 0.3s ease; - } - .ant-table-row { - cursor: pointer; - } - .ant-table-row > td.ant-table-cell { - padding: 8px; - padding-left: 16px; - font-size: 13px; - font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; - color: #5a656d; - .ant-typography { - font-size: 13px; - font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; - } - } +const Container = styled.div` + background-color: #ffffff; + padding: 1rem; `; const StyledTitle = styled.div` margin-bottom: 4px; font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; - font-size: 16px; font-weight: 500; - margin-left: 4px; - -`; - -const Container = styled.div` - padding: 16px; - background-color: #ffffff; + margin-bottom: 4px; + display: block; + font-size: 14px; + color: #5a656d; `; export const TagsContainer = styled.div` @@ -71,117 +58,125 @@ export const TagsContainer = styled.div` } `; +// Unified AG Grid Table Component for Configuration +const ConfigurationExampleTable: React.FC<{ data: QuestionSolution[] }> = ({ data }) => { + const [colDefs, setColDefs] = useState([]); + const [rowData, setRowData] = useState([]); + + // Simple cell renderer without click interactions + const TextCellRenderer = (params: ICellRendererParams) => { + const { value } = params; + if (!value) return ''; + + return ( +
+ {value} +
+ ); + }; + + useEffect(() => { + if (!isEmpty(data)) { + const columnDefs: ColDef[] = [ + { + field: 'question', + headerName: 'Prompts', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 300, + cellRenderer: TextCellRenderer, + wrapText: true, + autoHeight: false, + }, + { + field: 'solution', + headerName: 'Completions', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 300, + cellRenderer: TextCellRenderer, + wrapText: true, + autoHeight: false, + } + ]; + setColDefs(columnDefs); + setRowData(data); + } + }, [data]); + + const defaultColDef: ColDef = useMemo( + () => ({ + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 250, + }), + [] + ); + + let index = 0; + const getRowId = useCallback( + ({ data: rowData }: GetRowIdParams) => { + index++; + return toString(index); + }, + [] + ); -const ConfigurationTab: React.FC = ({ dataset }) => { - const topics = get(dataset, 'topics', []); + if (isEmpty(data)) { + return ( +
+ +
+ ); + } - const exampleColummns = [ - { - title: 'Prompts', - dataIndex: 'prompts', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.question} - }, - { - title: 'Completions', - dataIndex: 'completions', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.solution} - }, - ] + return ( +
+ +
+ ); +}; - const parameterColummns = [ - { - title: 'Temperature', - dataIndex: 'temperature', - ellipsis: true, - render: (temperature: number) => <>{temperature} - }, - { - title: 'Top K', - dataIndex: 'top_k', - ellipsis: true, - render: (top_k: number) => <>{top_k} - }, - { - title: 'Top P', - dataIndex: 'top_p', - ellipsis: true, - render: (top_p: number) => <>{top_p} - }, +const ConfigurationTab: React.FC = ({ dataset }) => { - ]; - return ( - - - - Custom Prompt - - {dataset?.custom_prompt} - - - - - {!isEmpty(topics) && - - - - Seed Instructions - - - {topics.map((tag: string) => ( - -
- {tag} -
- - ))} - - - - - } Examples - {dataset.technique === 'freeform' && } + {dataset.technique === 'freeform' && } {dataset.technique !== 'freeform' && - ({ - onClick: () => Modal.info({ - title: 'View Details', - content: , - icon: undefined, - maskClosable: false, - width: 1000 - }) - })} - rowKey={(_record, index) => `summary-examples-table-${index}`} - />} - - - - - - - Parameters - `parameters-table-${index}`} - /> + } diff --git a/app/client/src/pages/DatasetDetails/ExamplesSection.tsx b/app/client/src/pages/DatasetDetails/ExamplesSection.tsx index 9d1eb024..69bebb02 100644 --- a/app/client/src/pages/DatasetDetails/ExamplesSection.tsx +++ b/app/client/src/pages/DatasetDetails/ExamplesSection.tsx @@ -1,83 +1,141 @@ -import { Collapse, Flex, Modal, Table } from "antd"; -import styled from "styled-components"; -import { DatasetResponse } from "../../../api/Datasets/response"; -import { QuestionSolution } from "../../../pages/DataGenerator/types"; -import { Dataset } from "../../../pages/Evaluator/types"; +import { Collapse, Flex, Modal } from 'antd'; +import styled from 'styled-components'; +import isEmpty from 'lodash/isEmpty'; +import React, { useEffect, useMemo, useCallback, useState } from 'react'; +import { AgGridReact } from 'ag-grid-react'; +import { themeMaterial, ModuleRegistry, ClientSideRowModelModule, ValidationModule, TextFilterModule, NumberFilterModule, DateFilterModule, type ColDef, type GetRowIdFunc, type GetRowIdParams } from 'ag-grid-community'; +import toString from 'lodash/toString'; -import ExampleModal from "./ExampleModal"; -import FreeFormExampleTable from "../DataGenerator/FreeFormExampleTable"; +import { QuestionSolution } from '../DataGenerator/types'; +import FreeFormExampleTable from '../DataGenerator/FreeFormExampleTable'; +import ExampleModal from './ExampleModal'; +import { DatasetResponse } from "../../api/Datasets/response"; +import { Dataset } from "../Evaluator/types"; -const Panel = Collapse.Panel; +// Register AG Grid modules +ModuleRegistry.registerModules([ + TextFilterModule, + NumberFilterModule, + DateFilterModule, + ClientSideRowModelModule, + ValidationModule +]); +const { Panel } = Collapse; -const StyledTable = styled(Table)` - font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; - color: #5a656d; - .ant-table-thead > tr > th { - color: #5a656d; - border-bottom: 1px solid #eaebec; - font-weight: 500; - text-align: left; - // background: #ffffff; - border-bottom: 1px solid #eaebec; - transition: background 0.3s ease; - } - .ant-table-row { - cursor: pointer; - } - .ant-table-row > td.ant-table-cell { - padding: 8px; - padding-left: 16px; - font-size: 13px; +const Label = styled.div` + margin-bottom: 4px; font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; - color: #5a656d; - .ant-typography { - font-size: 13px; - font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; - } - } + font-weight: 500; + margin-bottom: 4px; + display: block; + font-size: 14px; + color: #5a656d; `; - - const StyledCollapse = styled(Collapse)` - .ant-collapse-content > .ant-collapse-content-box { - padding: 0; - } - .ant-collapse-item > .ant-collapse-header .ant-collapse-expand-icon { - height: 28px; - display: flex; - align-items: center; - padding-inline-end: 12px; - } + .ant-collapse-content > .ant-collapse-content-box { + padding: 0 !important; + } `; -const Label = styled.div` - font-size: 18px; - padding-top: 8px; -`; +// Unified AG Grid Table Component for Dataset Details +const DatasetExampleTable: React.FC<{ data: QuestionSolution[] }> = ({ data }) => { + const [colDefs, setColDefs] = useState([]); + const [rowData, setRowData] = useState([]); + + useEffect(() => { + if (!isEmpty(data)) { + const columnDefs: ColDef[] = [ + { + field: 'question', + headerName: 'Prompts', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 200, + wrapText: true, + autoHeight: true, + }, + { + field: 'solution', + headerName: 'Completions', + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 200, + wrapText: true, + autoHeight: true, + } + ]; + setColDefs(columnDefs); + setRowData(data); + } + }, [data]); + + const defaultColDef: ColDef = useMemo( + () => ({ + flex: 1, + filter: true, + sortable: true, + resizable: true, + minWidth: 170, + wrapText: true, + autoHeight: true, + }), + [] + ); + + let index = 0; + const getRowId = useCallback( + ({ data: rowData }: GetRowIdParams) => { + index++; + return toString(index); + }, + [] + ); + + const onRowClicked = useCallback((event: { data: QuestionSolution }) => { + const record = event.data; + Modal.info({ + title: 'View Details', + content: , + icon: undefined, + maskClosable: false, + width: 1000 + }); + }, []); + + return ( +
+ +
+ ); +}; export type DatasetDetailProps = { datasetDetails: DatasetResponse | Dataset; } const ExamplesSection= ({ datasetDetails }: DatasetDetailProps) => { - const { technique } = datasetDetails; - - const exampleCols = [ - { - title: 'Prompts', - dataIndex: 'prompts', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.question} - }, - { - title: 'Completions', - dataIndex: 'completions', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.solution} - }, - ] + // Handle both DatasetResponse and Dataset types + const technique = 'technique' in datasetDetails ? datasetDetails.technique : 'sft'; + const examples = datasetDetails.examples || []; return ( @@ -90,25 +148,10 @@ const ExamplesSection= ({ datasetDetails }: DatasetDetailProps) => { {technique === 'freeform' ? ( ) : - ({ - onClick: () => Modal.info({ - title: 'View Details', - content: , - icon: undefined, - maskClosable: false, - width: 1000 - }) - })} - rowKey={(_record, index) => `summary-examples-table-${index}`} - />} + } diff --git a/app/client/src/pages/Evaluator/types.ts b/app/client/src/pages/Evaluator/types.ts index 74d698f9..e44ea4bd 100644 --- a/app/client/src/pages/Evaluator/types.ts +++ b/app/client/src/pages/Evaluator/types.ts @@ -22,6 +22,7 @@ export interface Dataset { examples: Example[]; schema: string | null; total_count: number; + completed_rows: number | null; num_questions: number; job_id: string; job_name: string; diff --git a/app/client/src/pages/Home/DatasetsTab.tsx b/app/client/src/pages/Home/DatasetsTab.tsx index 508db33f..9ba15210 100644 --- a/app/client/src/pages/Home/DatasetsTab.tsx +++ b/app/client/src/pages/Home/DatasetsTab.tsx @@ -6,7 +6,7 @@ import { useDatasets } from './hooks'; import Loading from '../Evaluator/Loading'; import { Dataset } from '../Evaluator/types'; import Paragraph from 'antd/es/typography/Paragraph'; -import { JOB_EXECUTION_TOTAL_COUNT_THRESHOLD, TRANSLATIONS } from '../../constants'; +import { JOB_EXECUTION_TOTAL_COUNT_THRESHOLD } from '../../constants'; import DateTime from '../../components/DateTime/DateTime'; import DatasetActions from './DatasetActions'; import { sortItemsByKey } from '../../utils/sortutils'; @@ -15,6 +15,7 @@ import DatasetExportModal, { ExportResult } from '../../components/Export/Export import React from 'react'; import { JobStatus } from '../../types'; import JobStatusIcon from '../../components/JobStatus/jobStatusIcon'; +import { useUseCaseMapping } from '../../api/hooks'; const { Search } = Input; @@ -56,6 +57,7 @@ const StyledParagraph = styled(Paragraph)` const DatasetsTab: React.FC = () => { const { data, isLoading, isError, refetch, setSearchQuery, pagination } = useDatasets(); + const { getUseCaseName } = useUseCaseMapping(); const [notificationInstance, notificationContextHolder] = notification.useNotification(); const [exportResult, setExportResult] = React.useState(); const [toggleDatasetExportModal, setToggleDatasetExportModal] = React.useState(false); @@ -116,14 +118,14 @@ const DatasetsTab: React.FC = () => { dataIndex: 'generate_file_name', sorter: sortItemsByKey('generate_file_name'), width: 250, - render: (generate_file_name) => {generate_file_name} + render: (generate_file_name: string) => {generate_file_name} }, { key: 'model_id', title: 'Model', dataIndex: 'model_id', sorter: sortItemsByKey('model_id'), width: 250, - render: (modelId) => {modelId} + render: (modelId: string) => {modelId} }, { key: 'num_questions', title: 'Questions Per Topic', @@ -139,19 +141,27 @@ const DatasetsTab: React.FC = () => { align: 'center', sorter: sortItemsByKey('total_count'), width: 80 + }, { + key: 'completed_rows', + title: 'Completed Rows', + dataIndex: 'completed_rows', + align: 'center', + sorter: sortItemsByKey('completed_rows'), + width: 100, + render: (completed_rows: number | null) => <>{completed_rows != null ? completed_rows : 'N/A'} }, { key: 'use_case', title: 'Use Case', dataIndex: 'use_case', sorter: sortItemsByKey('use_case'), - render: (useCase) => TRANSLATIONS[useCase] + render: (useCase: string) => getUseCaseName(useCase) }, { key: 'timestamp', title: 'Creation Time', dataIndex: 'timestamp', defaultSortOrder: 'descend', sorter: sortItemsByKey('timestamp'), - render: (timestamp) => <>{timestamp == null ? 'N/A' : } + render: (timestamp: string | null) => <>{timestamp == null ? 'N/A' : } }, { key: '7', title: 'Actions', diff --git a/app/client/src/pages/Home/EvaluationsTab.tsx b/app/client/src/pages/Home/EvaluationsTab.tsx index 3ee69c8d..d6eb62fc 100644 --- a/app/client/src/pages/Home/EvaluationsTab.tsx +++ b/app/client/src/pages/Home/EvaluationsTab.tsx @@ -3,7 +3,7 @@ import { SyntheticEvent, useEffect } from "react"; import { Badge, Col, Flex, Input, notification, Row, Table, TableProps } from "antd"; import styled from "styled-components"; import Paragraph from 'antd/es/typography/Paragraph'; -import { JOB_EXECUTION_TOTAL_COUNT_THRESHOLD, TRANSLATIONS } from '../../constants'; +import { JOB_EXECUTION_TOTAL_COUNT_THRESHOLD } from '../../constants'; import { useEvaluations } from "./hooks"; import { Evaluation } from "./types"; import { sortItemsByKey } from "../../utils/sortutils"; @@ -15,6 +15,7 @@ import EvaluateActions from "./EvaluateActions"; import { getColorCode } from "../Evaluator/util"; import { JobStatus } from "../../types"; import JobStatusIcon from "../../components/JobStatus/jobStatusIcon"; +import { useUseCaseMapping } from "../../api/hooks"; const { Search } = Input; @@ -55,6 +56,7 @@ const StyledParagraph = styled(Paragraph)` const EvaluationsTab: React.FC = () => { const { data, isLoading, isError, refetch, setSearchQuery, pagination } = useEvaluations(); + const { getUseCaseName } = useUseCaseMapping(); useEffect(() => { if (isError) { @@ -99,20 +101,20 @@ const EvaluationsTab: React.FC = () => { key: 'average_score', title: 'Average Score', dataIndex: 'average_score', - render: (average_score) => , + render: (average_score: number) => , sorter: sortItemsByKey('average_score'), },{ key: 'use_case', title: 'Use Case', dataIndex: 'use_case', sorter: sortItemsByKey('use_case'), - render: (useCase) => {TRANSLATIONS[useCase]} + render: (useCase: string) => {getUseCaseName(useCase)} }, { key: 'timestamp', title: 'Create Time', dataIndex: 'timestamp', sorter: sortItemsByKey('timestamp'), - render: (timestamp) => + render: (timestamp: string) => }, { key: 'action', diff --git a/app/core/config.py b/app/core/config.py index 78a14f7a..ea70ffc2 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -826,7 +826,7 @@ class UseCaseMetadataEval(BaseModel): Cross-column guidelines: 1) Check for logical and realistic consistency and correlations between variables. Examples include but not limited to: - a) Grade/Sub-grade consistency: Sub-grade must match the grade (e.g., "B" grade → "B1" to "B5" possible subgrades). + a) Grade/Sub-grade consistency: Sub-grade must match the grade (e.g., "B" grade → "B1" to "B5"). b) Interest Rate vs Grade/Subgrade relationship: Higher subgrades (e.g., A5) could have higher `int_rate` than lower subgrades (e.g., A3). c) Mortgage Consistency: `mort_acc` should be 1 or more if `home_ownership` is `MORTGAGE`. d) Open Accounts: `open_acc` ≤ `total_acc`. diff --git a/app/core/database.py b/app/core/database.py index 57e286a9..f8fdaad8 100644 --- a/app/core/database.py +++ b/app/core/database.py @@ -820,6 +820,38 @@ def get_all_generate_metadata(self) -> List[Dict]: except Exception as e: print(f"Error retrieving all metadata: {str(e)}") return [] + + def get_paginated_generate_metadata_light(self, page: int, page_size: int) -> Tuple[int, List[Dict]]: + """Retrieve paginated metadata with only fields needed for list view""" + try: + with self.get_connection() as conn: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + # Get total count + count_query = "SELECT COUNT(*) FROM generation_metadata" + cursor.execute(count_query) + total_count = cursor.fetchone()[0] + + # Get only fields needed for list view + offset = (page - 1) * page_size + query = """ + SELECT + id, timestamp, display_name, generate_file_name, model_id, + num_questions, total_count, use_case, job_status, + local_export_path, hf_export_path, completed_rows + FROM generation_metadata + ORDER BY timestamp DESC + LIMIT ? OFFSET ? + """ + cursor.execute(query, (page_size, offset)) + + results = [dict(row) for row in cursor.fetchall()] + return total_count, results + + except Exception as e: + print(f"Error retrieving paginated metadata: {str(e)}") + return 0, [] def get_paginated_generate_metadata(self, page: int, page_size: int) -> Tuple[int, List[Dict]]: """Retrieve paginated metadata entries for generations""" diff --git a/app/main.py b/app/main.py index 88c6a9e5..9ad89967 100644 --- a/app/main.py +++ b/app/main.py @@ -986,7 +986,8 @@ async def get_generation_history( db_manager.update_job_statuses_generate(job_status_map) # Get paginated data - total_count, results = db_manager.get_paginated_generate_metadata(page, page_size) + #otal_count, results = db_manager.get_paginated_generate_metadata(page, page_size) + total_count, results = db_manager.get_paginated_generate_metadata_light(page, page_size) # Return in the structure expected by the frontend return { @@ -1154,10 +1155,23 @@ async def get_topics(use_case: UseCase): @app.get("/{use_case}/gen_examples") async def get_gen_examples(use_case: UseCase): - if use_case ==UseCase.CUSTOM: - return {"examples":[]} - else: - return {"examples": USE_CASE_CONFIGS[use_case].default_examples} + if use_case == UseCase.CUSTOM: + return {"examples": []} + else: + examples = USE_CASE_CONFIGS[use_case].default_examples + + # Transform field names for ticketing dataset to match frontend expectations + if use_case == UseCase.TICKETING_DATASET: + transformed_examples = [] + for example in examples: + transformed_example = { + "question": example.get("Prompt", ""), + "solution": example.get("Completion", "") + } + transformed_examples.append(transformed_example) + return {"examples": transformed_examples} + + return {"examples": examples} @app.get("/{use_case}/eval_examples") async def get_eval_examples(use_case: UseCase): diff --git a/tests/integration/test_synthesis_api.py b/tests/integration/test_synthesis_api.py index 34b81cf9..c474760e 100644 --- a/tests/integration/test_synthesis_api.py +++ b/tests/integration/test_synthesis_api.py @@ -48,8 +48,8 @@ def test_generate_endpoint_with_doc_paths(): assert "export_path" in res_json def test_generation_history(): - # Patch db_manager.get_paginated_generate_metadata to return dummy metadata with pagination info - db_manager.get_paginated_generate_metadata = lambda page, page_size: ( + # Patch db_manager.get_paginated_generate_metadata_light to return dummy metadata with pagination info + db_manager.get_paginated_generate_metadata_light = lambda page, page_size: ( 1, # total_count [{"generate_file_name": "qa_pairs_claude_20250210T170521148_test.json", "timestamp": "2024-02-10T12:00:00",