From 8f0d3bd689370225fd372a8605d901b8ff3fc859 Mon Sep 17 00:00:00 2001 From: Khauneesh-AI Date: Mon, 23 Jun 2025 17:51:53 +0530 Subject: [PATCH 01/26] added sds banner --- images/synthetic-data-studio-banner.svg | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 images/synthetic-data-studio-banner.svg diff --git a/images/synthetic-data-studio-banner.svg b/images/synthetic-data-studio-banner.svg new file mode 100644 index 00000000..6804615e --- /dev/null +++ b/images/synthetic-data-studio-banner.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + From d2fa5f41b78cac9a6c8a98f00677174728b994b4 Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Wed, 18 Jun 2025 16:34:51 -0700 Subject: [PATCH 02/26] DSE-45299: Fetching Templates from SDS API for SDS UI Add UseCaseSlector --- .../src/pages/DataGenerator/Configure.tsx | 20 +------ .../pages/DataGenerator/UseCaseSelector.tsx | 54 +++++++++++++++++++ app/client/src/pages/DataGenerator/hooks.ts | 26 ++++++++- app/client/src/types.ts | 9 +++- 4 files changed, 89 insertions(+), 20 deletions(-) create mode 100644 app/client/src/pages/DataGenerator/UseCaseSelector.tsx diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index a68bec84..bb7d42ea 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -10,6 +10,7 @@ import { MODEL_PROVIDER_LABELS } from './constants'; import { ModelProviders, ModelProvidersDropdownOpts } from './types'; import { useWizardCtx } from './utils'; import FileSelectorButton from './FileSelectorButton'; +import UseCaseSelector from './UseCaseSelector'; const StepContainer = styled(Flex)` @@ -224,24 +225,7 @@ const Configure = () => { {(formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) && - - - } + } {( formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || diff --git a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx new file mode 100644 index 00000000..558ee28d --- /dev/null +++ b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx @@ -0,0 +1,54 @@ +import { Form, Select } from "antd"; +import { FunctionComponent, useEffect, useState } from "react"; +import { useGetUseCases } from "./hooks"; +import { UseCase } from "../../types"; +import get from "lodash/get"; + +interface Props {} + + +const UseCaseSelector: FunctionComponent = () => { + const [useCases, setUseCases] = useState([]); + const useCasesReq = useGetUseCases(); + console.log('useCasesReq', useCasesReq); + + useEffect(() => { + if (useCasesReq.data) { + console.log('useCasesReq.data', useCasesReq.data); + let _useCases = get(useCasesReq, 'data.usecases', []); + _useCases = _useCases.map((useCase: any) => ({ + ...useCase, + label: useCase.name, + value: useCase.id + })); + console.log('_useCases', _useCases); + setUseCases(_useCases); + } + }, [useCasesReq.data]); + + + return ( + + + + ); +} + +export default UseCaseSelector; \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/hooks.ts b/app/client/src/pages/DataGenerator/hooks.ts index 2cb1dc7e..5bd40e57 100644 --- a/app/client/src/pages/DataGenerator/hooks.ts +++ b/app/client/src/pages/DataGenerator/hooks.ts @@ -244,4 +244,28 @@ export const useDatasetSize = ( isError, error }; - } \ No newline at end of file + } + + export const fetchUseCases = async () => { + const resp = await fetch(`${BASE_API_URL}/use-cases`, { + method: 'GET' + }); + const body = await resp.json(); + return body; +} + +export const useGetUseCases = () => { + const { data, isLoading, isError, error, isFetching } = useQuery( + { + queryKey: ['fetchUseCases', fetchUseCases], + queryFn: () => fetchUseCases(), + refetchOnWindowFocus: false, + } + ); + return { + data, + isLoading: isLoading || isFetching, + isError, + error + }; +} \ No newline at end of file diff --git a/app/client/src/types.ts b/app/client/src/types.ts index ee3381bf..295cccba 100644 --- a/app/client/src/types.ts +++ b/app/client/src/types.ts @@ -45,4 +45,11 @@ export const EXPORT_TYPE_LABELS: Record = { export type JobStatus = 'ENGINE_STOPPED' | 'ENGINE_SUCCEEDED' | 'ENGINE_TIMEDOUT' | 'ENGINE_SCHEDULING' | 'ENGINE_RUNNING' | 'null' | 'default'; -export const HuggingFaceIconUrl = "https://huggingface.co/front/assets/huggingface_logo-noborder.svg"; \ No newline at end of file +export const HuggingFaceIconUrl = "https://huggingface.co/front/assets/huggingface_logo-noborder.svg"; + +export interface UseCase { + name: string; + id: string; + label: string; + value: string; +} \ No newline at end of file From 246be13c601d2c6f02b42154f5f597281265d6cf Mon Sep 17 00:00:00 2001 From: Khauneesh-AI Date: Mon, 7 Jul 2025 14:38:51 +0530 Subject: [PATCH 03/26] changes to track completed rows in a complete or partial run --- .../2b4e8d9f6c3a_add_completed_rows.py | 30 +++++++++++++++++++ app/core/database.py | 19 +++++++----- app/migrations/alembic_schema_models.py | 1 + app/services/synthesis_service.py | 12 ++++---- 4 files changed, 49 insertions(+), 13 deletions(-) create mode 100644 alembic/versions/2b4e8d9f6c3a_add_completed_rows.py diff --git a/alembic/versions/2b4e8d9f6c3a_add_completed_rows.py b/alembic/versions/2b4e8d9f6c3a_add_completed_rows.py new file mode 100644 index 00000000..5f1d56d9 --- /dev/null +++ b/alembic/versions/2b4e8d9f6c3a_add_completed_rows.py @@ -0,0 +1,30 @@ +"""add_completed_rows + +Revision ID: 2b4e8d9f6c3a +Revises: 1a8fdc23eb6f +Create Date: 2025-01-18 10:30:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '2b4e8d9f6c3a' +down_revision: Union[str, None] = '1a8fdc23eb6f' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add completed_rows column to generation_metadata table + with op.batch_alter_table('generation_metadata', schema=None) as batch_op: + batch_op.add_column(sa.Column('completed_rows', sa.Integer(), nullable=True)) + + +def downgrade() -> None: + # Remove completed_rows column from generation_metadata table + with op.batch_alter_table('generation_metadata', schema=None) as batch_op: + batch_op.drop_column('completed_rows') \ No newline at end of file diff --git a/app/core/database.py b/app/core/database.py index f5c7682c..57e286a9 100644 --- a/app/core/database.py +++ b/app/core/database.py @@ -71,8 +71,9 @@ def init_db(self): job_id TEXT, job_name TEXT UNIQUE, job_status TEXT, - job_creator_name TEXT - + job_creator_name TEXT, + completed_rows INTEGER + ) """) @@ -163,8 +164,8 @@ def save_generation_metadata(self, metadata: Dict) -> int: custom_prompt, model_parameters, input_key, output_key, output_value, generate_file_name, display_name, local_export_path, hf_export_path, s3_export_path, num_questions, total_count, topics, examples, - schema, doc_paths, input_path, job_id, job_name, job_status, job_creator_name - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + schema, doc_paths, input_path, job_id, job_name, job_status, job_creator_name, completed_rows + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ values = ( @@ -194,7 +195,8 @@ def save_generation_metadata(self, metadata: Dict) -> int: metadata.get('job_id', None), metadata.get('job_name', None), metadata.get('job_status', None), - metadata.get('job_creator_name', None) + metadata.get('job_creator_name', None), + metadata.get('completed_rows', None) ) cursor.execute(query, values) @@ -212,7 +214,7 @@ def save_generation_metadata(self, metadata: Dict) -> int: print(f"Error saving metadata to database: {str(e)}") raise - def update_job_generate(self, job_name: str, generate_file_name: str, local_export_path: str, timestamp: str, job_status): + def update_job_generate(self, job_name: str, generate_file_name: str, local_export_path: str, timestamp: str, job_status, completed_rows): """Update job generate with retry mechanism""" max_retries = 3 retry_delay = 1 # seconds @@ -244,11 +246,12 @@ def update_job_generate(self, job_name: str, generate_file_name: str, local_expo SET generate_file_name = ?, local_export_path = ?, timestamp = ?, - job_status = ? + job_status = ?, + completed_rows = ? WHERE job_name = ? AND job_name IS NOT NULL AND job_name != '' - """, (generate_file_name, local_export_path, timestamp, job_status, job_name)) + """, (generate_file_name, local_export_path, timestamp, job_status, completed_rows,job_name)) rows_affected = cursor.rowcount conn.commit() diff --git a/app/migrations/alembic_schema_models.py b/app/migrations/alembic_schema_models.py index 3967a113..5eb735f2 100644 --- a/app/migrations/alembic_schema_models.py +++ b/app/migrations/alembic_schema_models.py @@ -35,6 +35,7 @@ class GenerationMetadataModel(Base): job_name = Column(Text, unique=True) job_status = Column(Text) job_creator_name = Column(Text) + completed_rows = Column(Integer) class EvaluationMetadataModel(Base): __tablename__ = 'evaluation_metadata' diff --git a/app/services/synthesis_service.py b/app/services/synthesis_service.py index 5aa25bb7..7f893f9e 100644 --- a/app/services/synthesis_service.py +++ b/app/services/synthesis_service.py @@ -1093,7 +1093,8 @@ def json_serializable(obj): 'input_path': input_path_str, 'input_key': request.input_key, 'output_key': request.output_key, - 'output_value': request.output_value + 'output_value': request.output_value, + 'completed_rows': len(final_output) if final_output else 0 } if is_demo: @@ -1109,7 +1110,7 @@ def json_serializable(obj): generate_file_name = os.path.basename(file_path) if final_output else '' final_output_path = file_path if final_output else '' - self.db.update_job_generate(job_name, generate_file_name, final_output_path, timestamp, job_status) + self.db.update_job_generate(job_name, generate_file_name, final_output_path, timestamp, job_status, len(final_output) if final_output else 0) self.db.backup_and_restore_db() return { "status": "completed" if final_output else "failed", @@ -1157,13 +1158,14 @@ def json_serializable(obj): if saved_partial_results: # Update with actual file information for partial results generate_file_name = os.path.basename(file_path) - final_output_path = file_path + final_output_path = file_path + completed_rows = len(final_output) if final_output else 0 else: # No results saved, use empty values generate_file_name = '' final_output_path = '' - - self.db.update_job_generate(job_name, generate_file_name, final_output_path, timestamp, job_status) + completed_rows = 0 + self.db.update_job_generate(job_name, generate_file_name, final_output_path, timestamp, job_status, completed_rows = completed_rows ) raise def get_health_check(self) -> Dict: From 5c435bdab21a6254890516e9cb8b87c36f14499f Mon Sep 17 00:00:00 2001 From: Khauneesh-AI Date: Mon, 7 Jul 2025 16:42:43 +0530 Subject: [PATCH 04/26] alembic upgrade made simpler and robust for new column updates --- app/migrations/alembic_manager.py | 119 ++++++++++++++++++------------ 1 file changed, 71 insertions(+), 48 deletions(-) diff --git a/app/migrations/alembic_manager.py b/app/migrations/alembic_manager.py index 51538c66..7e1b257a 100644 --- a/app/migrations/alembic_manager.py +++ b/app/migrations/alembic_manager.py @@ -1,61 +1,84 @@ # app/migrations/alembic_manager.py -from alembic.config import Config -from alembic import command -from alembic.script import ScriptDirectory -from alembic.runtime.migration import MigrationContext -from pathlib import Path -import os -from sqlalchemy import create_engine +import subprocess class AlembicMigrationManager: def __init__(self, db_path: str = None): - """Initialize Alembic with the same database path as DatabaseManager""" - self.app_path = Path(__file__).parent.parent.parent - - if db_path is None: - db_path = os.path.join(self.app_path, "metadata.db") - self.db_path = db_path - - # Initialize Alembic config - self.alembic_cfg = Config(str(self.app_path / "alembic.ini")) - self.alembic_cfg.set_main_option('script_location', str(self.app_path / "alembic")) - self.alembic_cfg.set_main_option('sqlalchemy.url', f'sqlite:///{db_path}') - - # Create engine for version checks - self.engine = create_engine(f'sqlite:///{db_path}') - - async def get_db_version(self) -> str: - """Get current database version""" - with self.engine.connect() as conn: - context = MigrationContext.configure(conn) - return context.get_current_revision() + """Initialize with database path (kept for interface compatibility)""" + self.db_path = db_path or "metadata.db" async def handle_database_upgrade(self) -> tuple[bool, str]: """ - Handle database migrations carefully to avoid disrupting existing data + Simple database migration using alembic upgrade head + No directory changes needed - already in project root """ try: - # First check if alembic_version table exists - try: - version = await self.get_db_version() - if version is None: - # Database exists but no alembic version - stamp current - command.stamp(self.alembic_cfg, "head") - return True, "Existing database stamped with current version" - except Exception: - # No alembic_version table - stamp current - command.stamp(self.alembic_cfg, "head") - return True, "Existing database stamped with current version" + # Run upgrade head - we're already in the right directory + result = subprocess.run( + ["alembic", "upgrade", "head"], + capture_output=True, + text=True, + check=True + ) - # Now check for and apply any new migrations - script = ScriptDirectory.from_config(self.alembic_cfg) - head_revision = script.get_current_head() + # Check if anything was actually upgraded + if "Running upgrade" in result.stdout: + return True, f"Database upgraded successfully: {result.stdout.strip()}" + else: + return True, "Database is already up to date" + + except subprocess.CalledProcessError as e: + error_msg = e.stderr or e.stdout or str(e) + return False, f"Database upgrade failed: {error_msg}" + except Exception as e: + return False, f"Error during database upgrade: {str(e)}" + + async def get_migration_status(self) -> dict: + """Get detailed migration status for debugging""" + try: + # Get current version + current_result = subprocess.run( + ["alembic", "current"], + capture_output=True, + text=True, + check=True + ) - if version != head_revision: - command.upgrade(self.alembic_cfg, "head") - return True, "Database schema updated successfully" + # Get head version + head_result = subprocess.run( + ["alembic", "show", "head"], + capture_output=True, + text=True, + check=True + ) - return True, "Database schema is up to date" - + return { + "current": current_result.stdout.strip(), + "head": head_result.stdout.strip(), + "status": "ready" + } + + except subprocess.CalledProcessError as e: + error_msg = e.stderr or e.stdout or str(e) + return {"error": f"Command failed: {error_msg}", "status": "error"} except Exception as e: - return False, f"Error during database upgrade: {str(e)}" \ No newline at end of file + return {"error": str(e), "status": "error"} + + async def get_current_version(self) -> str: + """Get current database version using alembic current command""" + try: + result = subprocess.run( + ["alembic", "current"], + capture_output=True, + text=True, + check=True + ) + + # Extract just the version ID from output like "2b4e8d9f6c3a (head)" + import re + match = re.search(r'([a-f0-9]{12})', result.stdout) + return match.group(1) if match else "none" + + except subprocess.CalledProcessError: + return "none" + except Exception: + return "unknown" \ No newline at end of file From d530647d07ae2944afccf4bcb0311f817e6f1c59 Mon Sep 17 00:00:00 2001 From: Khauneesh-AI Date: Mon, 7 Jul 2025 19:24:34 +0530 Subject: [PATCH 05/26] upgrade database process separated as a subprocess within same endpoint | This will enable newer migration easily without manual upgrade --- app/main.py | 23 ++++++++++++++++------- run_migrations.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 run_migrations.py diff --git a/app/main.py b/app/main.py index 1ce5d47a..3ec34cec 100644 --- a/app/main.py +++ b/app/main.py @@ -1420,13 +1420,22 @@ async def perform_upgrade(): # 2. Database migrations try: - db_success, db_message = await alembic_manager.handle_database_upgrade() - if db_success: - db_upgraded = True - messages.append(db_message) - else: - messages.append(f"Database upgrade failed: {db_message}") - raise HTTPException(status_code=500, detail=db_message) + # In your upgrade endpoint, you can add this debug line: + print(f"Current working directory: {os.getcwd()}") + print(f"Alembic.ini exists: {os.path.exists('alembic.ini')}") + print("--- Starting database migration via external script ---") + # Use `uv run` to ensure the script runs within the project's virtual environment + # This is more robust than just calling 'python' + result = subprocess.run( + ["uv", "run", "python", "run_migrations.py"], + capture_output=True, + text=True, + check=True # This will raise CalledProcessError on failure + ) + + print(result.stdout) # Log the output from the script + db_upgraded = True + messages.append("Database migration check completed successfully.") except Exception as e: messages.append(f"Database migration failed: {str(e)}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/run_migrations.py b/run_migrations.py new file mode 100644 index 00000000..dc83e493 --- /dev/null +++ b/run_migrations.py @@ -0,0 +1,33 @@ +import asyncio +import sys +from pathlib import Path + +# Ensure the 'app' directory is in the Python path +ROOT_DIR = Path(__file__).parent +APP_DIR = ROOT_DIR / "app" +sys.path.append(str(ROOT_DIR)) + +from app.migrations.alembic_manager import AlembicMigrationManager + +async def main(): + """ + Initializes the migration manager and runs the database upgrade. + This will always use the latest code from disk. + """ + print("--- Running dedicated migration script ---") + # Assumes your DB file is named metadata.db in the root + db_path = str(ROOT_DIR / "metadata.db") + alembic_manager = AlembicMigrationManager(db_path) + + success, message = await alembic_manager.handle_database_upgrade() + + if not success: + print(f"Migration Error: {message}") + # Exit with a non-zero status code to indicate failure + sys.exit(1) + + print(f"Migration Success: {message}") + print("--- Migration script finished ---") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file From 29187307143d02ee3bd071171955f60095590361 Mon Sep 17 00:00:00 2001 From: Khauneesh-AI Date: Mon, 7 Jul 2025 20:10:12 +0530 Subject: [PATCH 06/26] added ticketing dataset temmplate i.e. ala hello world template for synthetic Data studio --- app/core/config.py | 79 ++++++++++++++++++++++++++++++++++++++++++++-- app/main.py | 1 + 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/app/core/config.py b/app/core/config.py index 777a6e47..78a14f7a 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -17,6 +17,7 @@ class UseCase(str, Enum): LENDING_DATA = "lending_data" #HOUSING_DATA = "housing_data" CREDIT_CARD_DATA = "credit_card_data" + TICKETING_DATASET = "ticketing_dataset" class Technique(str, Enum): SFT = "sft" @@ -595,9 +596,46 @@ class UseCaseMetadataEval(BaseModel): """, - schema=None + schema=None ), -} + + UseCase.TICKETING_DATASET: UseCaseMetadata( + name="Ticketing Dataset", + description= "Synthetic dataset for ticketing system ", + topics=["Technical Issues", "Billing Queries", "Payment queries"], + default_examples=[ + { + "Prompt": "I have received this message that I owe $300 and I was instructed to pay the bill online. I already paid this amount and I am wondering why I received this message.", + "Completion": "report_payment_issue" + }, + { + "Prompt": "I will not be able to attend the presentation and would like to cancel my rsvp.", + "Completion": "cancel_ticket" + }, + { + "Prompt": "I am having questions regarding the exact time, location, and requirements of the event and would like to talk to customer service.", + "Completion": "Customer_service" + } + ] + , + prompt= """ + Generate authentic customer support ticket interactions that have a user query and system response. + For each user query, the system generates a keyword that is used to forward the user to the specific subsystem. + Requirements for user queries: + - Use professional, respectful language + - Follow standard customer service best practices + Each response should be a single id from the following list: + cancel_ticket,customer_service,report_payment_issue + Here are the explanations of the responses: + cancel_ticket means that the customer wants to cancel the ticket. + customer_service means that customer wants to talk to customer service. + report_payment_issue means that the customer is facing payment issues and wants to be forwarded to the billing department to resolve the issue. + + """, + schema=None + ) + } + USE_CASE_CONFIGS_EVALS = { @@ -916,6 +954,43 @@ class UseCaseMetadataEval(BaseModel): Give a score rating 1-10 for the given data. If there are more than 9 points to subtract use 1 as the absolute minimum scoring. List all justification as list. + """ + ), + UseCase.TICKETING_DATASET: UseCaseMetadataEval( + name="Ticketing Dataset", + default_examples=[ + { + "score": 5, + "justification": """ + The query is professionally written, respectful, and follows customer service best practices. + The response 'report_payment_issue' is one of the allowed keywords. + The matching between the query and response is perfect according to the provided definitions. + + """}, + { + "score": 3, + "justification": """ + The query is professionally written and respectful. + The response 'cancel_ticket' is one of the allowed keywords. + While the response uses a valid keyword, it doesn't match the most appropriate category for the specific query content. + """ + }, + + ], + prompt= """ + You are given a user query for a ticketing support system and the system responses which is a keyword that is used to forward the user to the specific subsystem. + Evaluate whether the queries: + - Use professional, respectful language + - Follow standard customer service best practices + Evaluate whether the responses use only one of the the following keywords: cancel_ticket,customer_service,report_payment_issue + Evaluate whether the solutions and responses are correctly matched based on the following definitions: + cancel_ticket means that the customer wants to cancel the ticket. + customer_service means that customer wants to talk to customer service. + report_payment_issue means that the customer is facing payment issues and wants to be forwarded to the billing department to resolve the issue. + Give a score of 1-5 based on the following instructions: + If the responses don’t match the four keywords give always value 1. + Rate the quality of the queries and responses based on the instructions give a rating between 1 to 5. + """ ) } diff --git a/app/main.py b/app/main.py index 3ec34cec..88c6a9e5 100644 --- a/app/main.py +++ b/app/main.py @@ -773,6 +773,7 @@ async def get_use_cases(): {"id": UseCase.CUSTOM, "name": "Custom"}, {"id": UseCase.LENDING_DATA, "name": "Lending Data"}, {"id": UseCase.CREDIT_CARD_DATA, "name": "Credit Card Data"}, + {"id": UseCase.TICKETING_DATASET, "name": "Ticketing Dataset"}, ] } From a4a16b929c627f11d4018f9ff9795f80d5f95ad1 Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Sun, 29 Jun 2025 22:54:31 -0700 Subject: [PATCH 07/26] DSE-45878: Add Mute Checkbox for the Welcome Page Fix Examples Table for Remote Templates Fix mute checkbox margin top Modify Examples Fix rendering examples based on the selected use case --- .../DataGenerator/CustomPromptButton.tsx | 1 - .../src/pages/DataGenerator/Examples.tsx | 34 ++++++---- .../pages/DataGenerator/UseCaseSelector.tsx | 5 +- app/client/src/pages/DataGenerator/hooks.ts | 62 ++++++++++++++++++- app/client/src/pages/DataGenerator/types.ts | 5 ++ app/client/src/pages/Home/WelcomePage.tsx | 13 +++- app/client/src/routes.tsx | 8 ++- 7 files changed, 106 insertions(+), 22 deletions(-) diff --git a/app/client/src/pages/DataGenerator/CustomPromptButton.tsx b/app/client/src/pages/DataGenerator/CustomPromptButton.tsx index d5781bef..e3fe10c4 100644 --- a/app/client/src/pages/DataGenerator/CustomPromptButton.tsx +++ b/app/client/src/pages/DataGenerator/CustomPromptButton.tsx @@ -59,7 +59,6 @@ const CustomPromptButton: React.FC = ({ model_id, inference_type, caii_en const [showModal, setShowModal] = useState(false); const [disabled, setDisabled] = useState(false); const custom_prompt_instructions = Form.useWatch('custom_prompt_instructions', { form, preserve: true }); - console.log('custom_prompt_instructions', custom_prompt_instructions); const mutation = useMutation({ mutationFn: fetchCustomPrompt diff --git a/app/client/src/pages/DataGenerator/Examples.tsx b/app/client/src/pages/DataGenerator/Examples.tsx index f5ce02ab..61d71acf 100644 --- a/app/client/src/pages/DataGenerator/Examples.tsx +++ b/app/client/src/pages/DataGenerator/Examples.tsx @@ -9,14 +9,14 @@ import { useMutation } from "@tanstack/react-query"; import { useFetchExamples } from '../../api/api'; import TooltipIcon from '../../components/TooltipIcon'; import PCModalContent from './PCModalContent'; -import { File, QuestionSolution, WorkflowType } from './types'; +import { ExampleType, File, QuestionSolution, WorkflowType } from './types'; import FileSelectorButton from './FileSelectorButton'; -import { fetchFileContent } from './hooks'; +import { fetchFileContent, getExampleType, useGetExamplesByUseCase } from './hooks'; import { useState } from 'react'; import FreeFormExampleTable from './FreeFormExampleTable'; -const { Title } = Typography; +const { Title, Text } = Typography; const Container = styled.div` padding-bottom: 10px ` @@ -48,10 +48,7 @@ const StyledContainer = styled.div` const MAX_EXAMPLES = 5; -enum ExampleType { - FREE_FORM = 'freeform', - PROMPT_COMPLETION = 'promptcompletion' -} + const Examples: React.FC = () => { const form = Form.useFormInstance(); @@ -90,13 +87,13 @@ const Examples: React.FC = () => { title: 'Prompts', dataIndex: 'question', ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.question} + render: (_text: QuestionSolution, record: QuestionSolution) => {record.question} }, { title: 'Completions', dataIndex: 'solution', ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => <>{record.solution} + render: (_text: QuestionSolution, record: QuestionSolution) => {record.solution} }, { title: 'Actions', @@ -178,13 +175,24 @@ const Examples: React.FC = () => { /> ) - }}, + } + }, ]; const dataSource = Form.useWatch('examples', form); - const { data: examples, loading: examplesLoading } = useFetchExamples(form.getFieldValue('use_case')); + const { examples, exmpleFormat, isLoading: examplesLoading } = + useGetExamplesByUseCase(form.getFieldValue('use_case')); + + // update examples if (!dataSource && examples) { - form.setFieldValue('examples', examples.examples) + form.setFieldValue('examples', examples) } + useEffect(() => { + if (!isEmpty(examples) && !isEmpty(exmpleFormat)) { + setExampleType(exmpleFormat as ExampleType); + form.setFieldValue('examples', examples || []); + } + }, [examples, exmpleFormat]); + const rowLimitReached = form.getFieldValue('examples')?.length === MAX_EXAMPLES; const workflowType = form.getFieldValue('workflow_type'); @@ -299,6 +307,8 @@ const Examples: React.FC = () => { {exampleType === ExampleType.FREE_FORM && !isEmpty(mutation.data) && } + {exampleType === ExampleType.FREE_FORM && form.getFieldValue('use_case') === 'lending_data' && + } {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && !isEmpty(values.examples) && } {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && isEmpty(values.examples) && diff --git a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx index 558ee28d..bb38a351 100644 --- a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx +++ b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx @@ -10,18 +10,15 @@ interface Props {} const UseCaseSelector: FunctionComponent = () => { const [useCases, setUseCases] = useState([]); const useCasesReq = useGetUseCases(); - console.log('useCasesReq', useCasesReq); useEffect(() => { if (useCasesReq.data) { - console.log('useCasesReq.data', useCasesReq.data); let _useCases = get(useCasesReq, 'data.usecases', []); _useCases = _useCases.map((useCase: any) => ({ ...useCase, label: useCase.name, value: useCase.id })); - console.log('_useCases', _useCases); setUseCases(_useCases); } }, [useCasesReq.data]); @@ -34,7 +31,7 @@ const UseCaseSelector: FunctionComponent = () => { rules={[ { required: true } ]} - tooltip='A specialize template for generating your dataset' + tooltip='A specialized template for generating your dataset' labelCol={{ span: 8 }} diff --git a/app/client/src/pages/DataGenerator/hooks.ts b/app/client/src/pages/DataGenerator/hooks.ts index 5bd40e57..c6069bb0 100644 --- a/app/client/src/pages/DataGenerator/hooks.ts +++ b/app/client/src/pages/DataGenerator/hooks.ts @@ -4,7 +4,8 @@ import toNumber from 'lodash/toNumber'; import isEmpty from 'lodash/isEmpty'; import isString from 'lodash/isString'; import { useMutation, useQuery } from '@tanstack/react-query'; -import { WorkflowType } from './types'; +import { ExampleType, WorkflowType } from './types'; +import { first } from 'lodash'; const BASE_API_URL = import.meta.env.VITE_AMP_URL; @@ -257,7 +258,7 @@ export const useDatasetSize = ( export const useGetUseCases = () => { const { data, isLoading, isError, error, isFetching } = useQuery( { - queryKey: ['fetchUseCases', fetchUseCases], + queryKey: ['useCases'], queryFn: () => fetchUseCases(), refetchOnWindowFocus: false, } @@ -268,4 +269,59 @@ export const useGetUseCases = () => { isError, error }; -} \ No newline at end of file +} + +export const fetchExamplesByUseCase = async (use_case: string) => { + const resp = await fetch(`${BASE_API_URL}/${isEmpty(use_case) ? 'custom' : use_case}/gen_examples`, { + method: 'GET' + }); + const body = await resp.json(); + return body; +} + +export const useGetExamplesByUseCase = (use_case: string) => { + const { data, isLoading, isError, error, isFetching } = useQuery( + { + queryKey: ['fetchUseCaseTopics', fetchExamplesByUseCase], + queryFn: () => fetchExamplesByUseCase(use_case), + refetchOnWindowFocus: false, + } + ); + + if (isError) { + notification.error({ + message: 'Error', + description: `An error occurred while fetching the use case examples.\n ${error?.message}` + }); + } + + + let examples = []; + let exmpleFormat: ExampleType | null = null; + if (!isEmpty(data) && !isEmpty(data?.examples)) { + examples = get(data, 'examples', []); + exmpleFormat = getExampleType(examples); + } + + return { + data, + isLoading: isLoading || isFetching, + isError, + error, + examples, + exmpleFormat + }; +} + +export const getExampleType = (data: object[]) => { + if (!isEmpty(data)) { + const row = first(data); + const keys = Object.keys(row as object); + if (keys.length === 2) { + return ExampleType.PROMPT_COMPLETION; + } + return ExampleType.FREE_FORM; + } + return null; +} + diff --git a/app/client/src/pages/DataGenerator/types.ts b/app/client/src/pages/DataGenerator/types.ts index cb029ed2..73c64b20 100644 --- a/app/client/src/pages/DataGenerator/types.ts +++ b/app/client/src/pages/DataGenerator/types.ts @@ -119,4 +119,9 @@ export enum TechniqueType { SFT = 'sft', CUSTOME_WORKFLOW = 'custom_workflow', FREE_FORM = 'freeform' +} + +export enum ExampleType { + FREE_FORM = 'freeform', + PROMPT_COMPLETION = 'promptcompletion' } \ No newline at end of file diff --git a/app/client/src/pages/Home/WelcomePage.tsx b/app/client/src/pages/Home/WelcomePage.tsx index 82498ab2..135f70dc 100644 --- a/app/client/src/pages/Home/WelcomePage.tsx +++ b/app/client/src/pages/Home/WelcomePage.tsx @@ -1,10 +1,12 @@ -import { Button, Col, Flex, Layout, Row, Image } from 'antd'; +import toString from 'lodash/toString'; +import { Button, Col, Flex, Layout, Row, Image, Checkbox } from 'antd'; import React from 'react'; import styled from 'styled-components'; import SDGIcon from '../../assets/sdg-landing.svg'; import LightBulbIcon from '../../assets/ic-lightbulb.svg'; import QueryPromptIcon from '../../assets/ic-query-prompt.svg'; import NumbersIcon from '../../assets/ic-numbers.svg'; +import { CheckboxChangeEvent } from 'antd/es/checkbox'; const { Content } = Layout; @@ -107,6 +109,11 @@ const InfoSection = styled.div` const WelcomePage: React.FC = () => { + const onChange = (e: CheckboxChangeEvent) => { + const checked = e.target.checked; + window.localStorage.setItem('sds_mute_welcome_page', toString(checked)); + } + return ( @@ -148,6 +155,10 @@ const WelcomePage: React.FC = () => {
+ +
+ {`Don't show me this again`} +
diff --git a/app/client/src/routes.tsx b/app/client/src/routes.tsx index 853257c5..ee600017 100644 --- a/app/client/src/routes.tsx +++ b/app/client/src/routes.tsx @@ -12,6 +12,10 @@ import EvaluationDetailsPage from "./pages/EvaluationDetails/EvaluationDetailsPa //import TelemetryDashboard from "./components/TelemetryDashboard"; +const isWelcomePageMuted = () => { + return window.localStorage.getItem('sds_mute_welcome_page') === 'true'; +} + const router = createBrowserRouter([ { path: '/', @@ -19,7 +23,9 @@ const router = createBrowserRouter([ children: [ { path: '/', // Redirect root to Pages.WELCOME - element: , + element: isWelcomePageMuted() ? : + , + errorElement: }, { path: Pages.HOME, From 5a88a567d5a465e2cc4202ae1e7c674fa6850404 Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Tue, 8 Jul 2025 19:55:27 -0700 Subject: [PATCH 08/26] DSE-45114: Add Data Augmentation Card This PR contains following changes: - Adds new card for Data Augmentation to the Home Page - Hide Workflow Type for the Data Augmentation --- .../src/assets/ic-data-augmentation.svg | 13 +++++++++++ .../src/pages/DataGenerator/Configure.tsx | 22 +++++++++++++++++-- .../src/pages/DataGenerator/DataGenerator.tsx | 9 ++++++-- app/client/src/pages/DataGenerator/utils.ts | 13 +++++++++++ app/client/src/pages/Home/HomePage.tsx | 20 +++++++++++++++++ app/client/src/routes.tsx | 10 +++++++-- app/client/src/types.ts | 6 +++++ 7 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 app/client/src/assets/ic-data-augmentation.svg diff --git a/app/client/src/assets/ic-data-augmentation.svg b/app/client/src/assets/ic-data-augmentation.svg new file mode 100644 index 00000000..38e47ea7 --- /dev/null +++ b/app/client/src/assets/ic-data-augmentation.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index bb7d42ea..28bd3a45 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -8,9 +8,12 @@ import { File, WorkflowType } from './types'; import { useFetchModels } from '../../api/api'; import { MODEL_PROVIDER_LABELS } from './constants'; import { ModelProviders, ModelProvidersDropdownOpts } from './types'; -import { useWizardCtx } from './utils'; +import { getWizardModel, getWizardModeType, useWizardCtx } from './utils'; import FileSelectorButton from './FileSelectorButton'; import UseCaseSelector from './UseCaseSelector'; +import { useLocation } from 'react-router-dom'; +import { WizardModeType } from '../../types'; +import { get } from 'lodash'; const StepContainer = styled(Flex)` @@ -39,7 +42,7 @@ export const USECASE_OPTIONS = [ export const WORKFLOW_OPTIONS = [ { label: 'Supervised Fine-Tuning', value: 'supervised-fine-tuning' }, { label: 'Custom Data Generation', value: 'custom' }, - { label: 'Freeform Data Generation', value: 'freeform' } + // { label: 'Freeform Data Generation', value: 'freeform' } ]; export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [ @@ -48,6 +51,18 @@ export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [ ]; const Configure = () => { + const location = useLocation(); + const [wizardModeType, setWizardModeType] = useState(getWizardModeType(location)); + + useEffect(() => { + if (wizardModeType === WizardModeType.DATA_AUGMENTATION) { + setWizardModeType(WizardModeType.DATA_AUGMENTATION); + form.setFieldValue('workflow_type', 'freeform'); + } else { + setWizardModeType(WizardModeType.DATA_GENERATION); + } + }, [location, wizardModeType]); + const form = Form.useFormInstance(); const formData = Form.useWatch((values) => values, form); const { setIsStepValid } = useWizardCtx(); @@ -141,8 +156,10 @@ const Configure = () => { label='Model Provider' rules={[{ required: true }]} labelCol={labelCol} + shouldUpdate > form.setFieldValue('model_id', undefined)} placeholder={'Select a model provider'} > @@ -227,7 +210,6 @@ const Configure = () => { label='Workflow' tooltip='A specialized workflow for your dataset' labelCol={labelCol} - hidden={wizardModeType === WizardModeType.DATA_AUGMENTATION} shouldUpdate rules={[ { required: true } diff --git a/app/client/src/pages/DataGenerator/DataGenerator.tsx b/app/client/src/pages/DataGenerator/DataGenerator.tsx index f44bdb7e..bbf9b71f 100644 --- a/app/client/src/pages/DataGenerator/DataGenerator.tsx +++ b/app/client/src/pages/DataGenerator/DataGenerator.tsx @@ -1,6 +1,6 @@ import isEmpty from 'lodash/isEmpty'; import isString from 'lodash/isString'; -import { FunctionComponent, useEffect, useRef, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { useLocation, useParams } from 'react-router-dom'; import { Button, Flex, Form, Layout, Steps } from 'antd'; @@ -20,15 +20,10 @@ import { DataGenWizardSteps, WizardStepConfig, WorkflowType } from './types'; import { WizardCtx } from './utils'; import { fetchDatasetDetails, useGetDatasetDetails } from '../DatasetDetails/hooks'; import { useMutation } from '@tanstack/react-query'; -import { WizardModeType } from '../../types'; const { Content } = Layout; // const { Title } = Typography; -interface Props { - mode?: WizardModeType; -} - const StyledTitle = styled.div` margin-top: 10px; font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; @@ -100,7 +95,7 @@ const steps: WizardStepConfig[] = [ /** * Wizard component for Synthetic Data Generation workflow */ -const DataGenerator: FunctionComponent = () => { +const DataGenerator = () => { const [current, setCurrent] = useState(0); const [maxStep, setMaxStep] = useState(0); const [isStepValid, setIsStepValid] = useState(false); diff --git a/app/client/src/pages/DataGenerator/utils.ts b/app/client/src/pages/DataGenerator/utils.ts index 699ce95e..2e10b648 100644 --- a/app/client/src/pages/DataGenerator/utils.ts +++ b/app/client/src/pages/DataGenerator/utils.ts @@ -3,7 +3,6 @@ import { WizardCtxObj } from './types'; import moment from 'moment'; import toString from 'lodash/toString'; import { File } from './types'; -import { WizardModeType } from '../../types'; export const WizardCtx = createContext(null); export const useWizardCtx = (): WizardCtxObj => { @@ -106,15 +105,3 @@ export const getHttpStatusCodeVerb = (statusCode: number) => { return null; } }; - -export const getWizardModeType = (location: any) => { - const pathname = location?.pathname || ''; - switch (pathname) { - case '/data-augmentation': - return WizardModeType.DATA_AUGMENTATION; - case '/data-generator': - return WizardModeType.DATA_GENERATION; - default: - return null; - } -} diff --git a/app/client/src/pages/Home/HomePage.tsx b/app/client/src/pages/Home/HomePage.tsx index d92197a4..231ba64d 100644 --- a/app/client/src/pages/Home/HomePage.tsx +++ b/app/client/src/pages/Home/HomePage.tsx @@ -7,7 +7,6 @@ import EvaluationsTab from './EvaluationsTab'; import DatasetIcon from '../../assets/ic-datasets.svg'; import ArrowRightIcon from '../../assets/ic-arrow-right.svg'; import EvaluateIcon from '../../assets/ic-evaluations.svg'; -import DataAugmentationIcon from '../../assets/ic-data-augmentation.svg'; import EvaluateButton from './EvaluateButton'; import ExportsTab from './ExportsTab'; @@ -117,25 +116,6 @@ const HomePage: React.FC = () => { - -
- Datasets -
-
-
Data Augmentation
-
-

Generate multi-dimension datasets using LLM custom prompts

-
-
-
-
- -
-
-
Datasets diff --git a/app/client/src/routes.tsx b/app/client/src/routes.tsx index 59e20a38..ee600017 100644 --- a/app/client/src/routes.tsx +++ b/app/client/src/routes.tsx @@ -2,7 +2,7 @@ import { Navigate, createBrowserRouter } from "react-router-dom"; import Layout from "./Container"; import DataGenerator from "./pages/DataGenerator"; import HomePage from "./pages/Home"; -import { Pages, WizardModeType } from "./types"; +import { Pages } from "./types"; import EvaluatorPage from "./pages/Evaluator"; import ReevaluatorPage from "./pages/Evaluator/ReevaluatorPage"; import DatasetDetailsPage from "./pages/DatasetDetails/DatasetDetailsPage"; @@ -35,13 +35,7 @@ const router = createBrowserRouter([ }, { path: Pages.GENERATOR, - element: , - errorElement: , - loader: async () => null - }, - { - path: Pages.DATA_AUGMENTATION, - element: , + element: , errorElement: , loader: async () => null }, diff --git a/app/client/src/types.ts b/app/client/src/types.ts index 24908745..295cccba 100644 --- a/app/client/src/types.ts +++ b/app/client/src/types.ts @@ -1,6 +1,5 @@ export enum Pages { GENERATOR = 'data-generator', - DATA_AUGMENTATION = 'data-augmentation', REGENERATE = 're-generate', EVALUATOR = 'evaluator', HISTORY = 'history', @@ -53,9 +52,4 @@ export interface UseCase { id: string; label: string; value: string; -} - -export enum WizardModeType { - DATA_GENERATION = 'data-generation', - DATA_AUGMENTATION = 'data-augmention' } \ No newline at end of file From 1534f9d9a7d5b56c9a9857ec8d229434be677943 Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Tue, 8 Jul 2025 19:55:27 -0700 Subject: [PATCH 12/26] DSE-45114: Add Data Augmentation Card This PR contains following changes: - Adds new card for Data Augmentation to the Home Page - Hide Workflow Type for the Data Augmentation --- .../src/assets/ic-data-augmentation.svg | 13 +++++++++++ .../src/pages/DataGenerator/Configure.tsx | 22 +++++++++++++++++-- .../src/pages/DataGenerator/DataGenerator.tsx | 9 ++++++-- app/client/src/pages/DataGenerator/utils.ts | 13 +++++++++++ app/client/src/pages/Home/HomePage.tsx | 20 +++++++++++++++++ app/client/src/routes.tsx | 10 +++++++-- app/client/src/types.ts | 6 +++++ 7 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 app/client/src/assets/ic-data-augmentation.svg diff --git a/app/client/src/assets/ic-data-augmentation.svg b/app/client/src/assets/ic-data-augmentation.svg new file mode 100644 index 00000000..38e47ea7 --- /dev/null +++ b/app/client/src/assets/ic-data-augmentation.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index bb7d42ea..28bd3a45 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -8,9 +8,12 @@ import { File, WorkflowType } from './types'; import { useFetchModels } from '../../api/api'; import { MODEL_PROVIDER_LABELS } from './constants'; import { ModelProviders, ModelProvidersDropdownOpts } from './types'; -import { useWizardCtx } from './utils'; +import { getWizardModel, getWizardModeType, useWizardCtx } from './utils'; import FileSelectorButton from './FileSelectorButton'; import UseCaseSelector from './UseCaseSelector'; +import { useLocation } from 'react-router-dom'; +import { WizardModeType } from '../../types'; +import { get } from 'lodash'; const StepContainer = styled(Flex)` @@ -39,7 +42,7 @@ export const USECASE_OPTIONS = [ export const WORKFLOW_OPTIONS = [ { label: 'Supervised Fine-Tuning', value: 'supervised-fine-tuning' }, { label: 'Custom Data Generation', value: 'custom' }, - { label: 'Freeform Data Generation', value: 'freeform' } + // { label: 'Freeform Data Generation', value: 'freeform' } ]; export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [ @@ -48,6 +51,18 @@ export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [ ]; const Configure = () => { + const location = useLocation(); + const [wizardModeType, setWizardModeType] = useState(getWizardModeType(location)); + + useEffect(() => { + if (wizardModeType === WizardModeType.DATA_AUGMENTATION) { + setWizardModeType(WizardModeType.DATA_AUGMENTATION); + form.setFieldValue('workflow_type', 'freeform'); + } else { + setWizardModeType(WizardModeType.DATA_GENERATION); + } + }, [location, wizardModeType]); + const form = Form.useFormInstance(); const formData = Form.useWatch((values) => values, form); const { setIsStepValid } = useWizardCtx(); @@ -141,8 +156,10 @@ const Configure = () => { label='Model Provider' rules={[{ required: true }]} labelCol={labelCol} + shouldUpdate > - + } {formData?.workflow_type === WorkflowType.CUSTOM_DATA_GENERATION && diff --git a/app/client/src/pages/DataGenerator/DataGenerator.tsx b/app/client/src/pages/DataGenerator/DataGenerator.tsx index f44bdb7e..e4c6196d 100644 --- a/app/client/src/pages/DataGenerator/DataGenerator.tsx +++ b/app/client/src/pages/DataGenerator/DataGenerator.tsx @@ -178,7 +178,7 @@ const DataGenerator: FunctionComponent = () => { return ( - {'Synthetic Dataset Studio'} + {'Configure Synthetic Dataset'} { +const Examples: FunctionComponent = () => { const form = Form.useFormInstance(); - const [exampleType, setExampleType] = useState(ExampleType.PROMPT_COMPLETION); - + const [records, setRecords] = useState[]>([]); + const workflowType = form.getFieldValue('workflow_type'); + const { examples, isLoading: examplesLoading, refetch } = + useGetExamplesByUseCase(form.getFieldValue('use_case')); + const mutation = useMutation({ mutationFn: fetchFileContent }); - const values = form.getFieldsValue(true) useEffect(() => { const example_path = form.getFieldValue('example_path'); - if (!isEmpty(example_path)) { mutation.mutate({ path: example_path }); } - - if (form.getFieldValue('workflow_type') === 'freeform') { - setExampleType(ExampleType.FREE_FORM); - } - - - }, [form.getFieldValue('example_path'), form.getFieldValue('workflow_type')]); - useEffect(() => { + useEffect(() => { + console.log('------------------> useEffect') if (!isEmpty(mutation.data)) { form.setFieldValue('examples', mutation.data); + if (!isEqual(mutation.data, records)) { + setRecords(mutation.data); + } + + } else if (Array.isArray(examples) && !isEqual(examples, records)) { + form.setFieldValue('examples', examples); + setRecords(examples); } - }, [mutation.data]); - - const columns = [ - { - title: 'Prompts', - dataIndex: 'question', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => {record.question} - }, - { - title: 'Completions', - dataIndex: 'solution', - ellipsis: true, - render: (_text: QuestionSolution, record: QuestionSolution) => {record.solution} - }, - { - title: 'Actions', - key: 'actions', - width: 130, - render: (_text: QuestionSolution, record: QuestionSolution, index: number) => { - const { question, solution } = record; - const editRow = (data: QuestionSolution) => { - const updatedExamples = [...form.getFieldValue('examples')]; - updatedExamples.splice(index, 1, data); - form.setFieldValue('examples', updatedExamples); - Modal.destroyAll() - } - const deleteRow = () => { - const updatedExamples = [...form.getFieldValue('examples')]; - updatedExamples.splice(index, 1); - form.setFieldValue('examples', updatedExamples); - } - return ( - - - - - ), - maskClosable: true, - width: 1000 - }) - }} - /> - - ) - } - }, - ]; - const dataSource = Form.useWatch('examples', form); - const { examples, exmpleFormat, isLoading: examplesLoading } = - useGetExamplesByUseCase(form.getFieldValue('use_case')); + }, [mutation.data, examples]); - // update examples - if (!dataSource && examples) { - form.setFieldValue('examples', examples) - } - useEffect(() => { - if (!isEmpty(examples) && !isEmpty(exmpleFormat)) { - setExampleType(exmpleFormat as ExampleType); - form.setFieldValue('examples', examples || []); - } - }, [examples, exmpleFormat]); - const rowLimitReached = form.getFieldValue('examples')?.length === MAX_EXAMPLES; - const workflowType = form.getFieldValue('workflow_type'); const onAddFiles = (files: File[]) => { if (!isEmpty (files)) { @@ -207,7 +97,6 @@ const Examples: React.FC = () => { ...values, example_path: get(file, '_path') }); - setExampleType(ExampleType.FREE_FORM); } } @@ -215,8 +104,16 @@ const Examples: React.FC = () => { span: 10 }; + const showEmptyState = workflowType === WorkflowType.FREE_FORM_DATA_GENERATION && + isEmpty(mutation.data) && + records.length === 0; + + console.log('examples', form.getFieldValue('examples')); + console.log('records', records); + return ( + {examplesLoading && }
@@ -242,12 +139,10 @@ const Examples: React.FC = () => { } - - {exampleType !== ExampleType.FREE_FORM && } - - {exampleType !== ExampleType.FREE_FORM && - - - } +
- {exampleType === ExampleType.FREE_FORM && !isEmpty(mutation.data) && - } - {exampleType === ExampleType.FREE_FORM && form.getFieldValue('use_case') === 'lending_data' && - } - {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && !isEmpty(values.examples) && - } - {exampleType === ExampleType.FREE_FORM && isEmpty(mutation.data) && isEmpty(values.examples) && + {!isEmpty(records) && } + {showEmptyState && ( - - - } - imageStyle={{ - height: 60, - marginBottom: 24 - }} - description={ - <> -

- Upload a JSON file containing examples -

-

- {'Examples should be in the format of a JSON array containing array of key & value pairs. The key should be the column name and the value should be the cell value.'} -

- - } - > - - } - {exampleType !== ExampleType.FREE_FORM && - - ({ - onClick: () => Modal.info({ - title: 'View Details', - content: , - icon: undefined, - maskClosable: true, - width: 1000 - }) - })} - rowClassName={() => 'hover-pointer'} - rowKey={(_record, index) => `examples-table-${index}`} - /> - } - + image={ + + + + } + imageStyle={{ + height: 60, + marginBottom: 24 + }} + description={ + <> +

+ {`Upload a JSON file containing examples`} +

+

+ {'Examples should be in the format of a JSON array containing array of key & value pairs. The key should be the column name and the value should be the cell value.'} +

+ + } + /> + )} ) }; diff --git a/app/client/src/pages/DataGenerator/FileSelectorButton.tsx b/app/client/src/pages/DataGenerator/FileSelectorButton.tsx index b8e6f880..a75a301b 100644 --- a/app/client/src/pages/DataGenerator/FileSelectorButton.tsx +++ b/app/client/src/pages/DataGenerator/FileSelectorButton.tsx @@ -9,9 +9,10 @@ interface Props { onAddFiles: (files: File[]) => void; workflowType: WorkflowType; label?: string; + allowFileTypes?: string[]; } -const FileSelectorButton: React.FC = ({ onAddFiles, workflowType, label }) => { +const FileSelectorButton: React.FC = ({ onAddFiles, workflowType, label, allowFileTypes }) => { const [showModal, setShowModal] = useState(false); const [selectedFiles, setSelectedFiles] = useState([]) @@ -43,7 +44,7 @@ const FileSelectorButton: React.FC = ({ onAddFiles, workflowType, label } onOk={() => onFinish()} width="60%" > - + )} diff --git a/app/client/src/pages/DataGenerator/FilesTable.tsx b/app/client/src/pages/DataGenerator/FilesTable.tsx index 5636655b..0fadbf7a 100644 --- a/app/client/src/pages/DataGenerator/FilesTable.tsx +++ b/app/client/src/pages/DataGenerator/FilesTable.tsx @@ -3,6 +3,7 @@ import filter from 'lodash/filter'; import clone from 'lodash/clone'; import set from 'lodash/set'; import forEach from 'lodash/forEach'; +import isString from 'lodash/isString'; import React, { useEffect, useState } from 'react'; import { Badge, Breadcrumb, Button, Col, Flex, List, Popover, Row, Table, Tooltip, Typography } from 'antd'; import styled from 'styled-components'; @@ -19,6 +20,7 @@ interface Props { workflowType: WorkflowType; files: File[]; onSelectedRows: (selectedRows: File[]) => void; + allowFileTypes?: string[]; } const StyledTable = styled(Table)` @@ -78,8 +80,14 @@ export const getSelectedRows = (fileSelectionMap: FileSelectionMap) => { return rows; } +export function getFileExtension(filename: string): string | null { + const lastDot = filename.lastIndexOf('.'); + if (lastDot === -1 || lastDot === 0) return null; + return filename.slice(lastDot + 1).toLowerCase(); +} + -const FilesTable: React.FC = ({ onSelectedRows, workflowType }) => { +const FilesTable: React.FC = ({ onSelectedRows, workflowType, allowFileTypes }) => { const [paths, setPaths] = useState(null); const [path, setPath] = useState(null); const [selectedRowKeys, setSelectedRowKeys] = useState([]); @@ -106,7 +114,12 @@ const FilesTable: React.FC = ({ onSelectedRows, workflowType }) => { if (isDirectory(record)) { return true; } - + if (Array.isArray(allowFileTypes) && !isEmpty(allowFileTypes)) { + const extension = getFileExtension(record.name); + if (isString(extension)) { + return !allowFileTypes.includes(extension as string); + } + } if (workflowType === WorkflowType.SUPERVISED_FINE_TUNING) { return !endsWith(record.name, '.pdf'); } else if (workflowType === WorkflowType.CUSTOM_DATA_GENERATION) { diff --git a/app/client/src/pages/DataGenerator/ImportExamples.tsx b/app/client/src/pages/DataGenerator/ImportExamples.tsx new file mode 100644 index 00000000..f3645597 --- /dev/null +++ b/app/client/src/pages/DataGenerator/ImportExamples.tsx @@ -0,0 +1,12 @@ +import React from 'react'; + +const ImportExamples: React.FC = () => { + return ( +
+

Import Examples

+

This is a placeholder for importing examples. Functionality will be added soon.

+
+ ); +}; + +export default ImportExamples; \ No newline at end of file diff --git a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx index bb38a351..1107bc91 100644 --- a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx +++ b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx @@ -1,13 +1,15 @@ -import { Form, Select } from "antd"; +import { Form, FormInstance, Select } from "antd"; import { FunctionComponent, useEffect, useState } from "react"; import { useGetUseCases } from "./hooks"; import { UseCase } from "../../types"; import get from "lodash/get"; -interface Props {} +interface Props { + form: FormInstance; +} -const UseCaseSelector: FunctionComponent = () => { +const UseCaseSelector: FunctionComponent = ({ form }) => { const [useCases, setUseCases] = useState([]); const useCasesReq = useGetUseCases(); @@ -23,6 +25,15 @@ const UseCaseSelector: FunctionComponent = () => { } }, [useCasesReq.data]); + const onChange = (value: string) => { + console.log('value', value); + form.setFieldValue('use_case', value); + if (value !== 'custom') { + form.setFieldValue('example_path', null); + form.setFieldValue('examples', []); + } + } + return ( = () => { }} shouldUpdate > - {useCases.map(option => {option.label} diff --git a/app/client/src/pages/DataGenerator/hooks.ts b/app/client/src/pages/DataGenerator/hooks.ts index c6069bb0..2980205b 100644 --- a/app/client/src/pages/DataGenerator/hooks.ts +++ b/app/client/src/pages/DataGenerator/hooks.ts @@ -280,7 +280,7 @@ export const fetchExamplesByUseCase = async (use_case: string) => { } export const useGetExamplesByUseCase = (use_case: string) => { - const { data, isLoading, isError, error, isFetching } = useQuery( + const { data, isLoading, isError, error, isFetching, refetch } = useQuery( { queryKey: ['fetchUseCaseTopics', fetchExamplesByUseCase], queryFn: () => fetchExamplesByUseCase(use_case), @@ -309,7 +309,8 @@ export const useGetExamplesByUseCase = (use_case: string) => { isError, error, examples, - exmpleFormat + exmpleFormat, + refetch }; } diff --git a/app/client/src/pages/Datasets/DatasetActions.tsx b/app/client/src/pages/Datasets/DatasetActions.tsx new file mode 100644 index 00000000..f33b9f7f --- /dev/null +++ b/app/client/src/pages/Datasets/DatasetActions.tsx @@ -0,0 +1,148 @@ +import isEmpty from 'lodash/isEmpty'; +import React from 'react'; +import { Button, Dropdown, Flex, MenuProps, Modal, Space, Typography } from "antd"; +import { Dataset } from "../Evaluator/types"; +import styled from "styled-components"; +import { ExportOutlined, FolderViewOutlined, ThunderboltOutlined } from '@ant-design/icons'; +import FindInPageIcon from '@mui/icons-material/FindInPage'; +import QueryStatsIcon from '@mui/icons-material/QueryStats'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import DeleteIcon from '@mui/icons-material/Delete'; +import { Link } from "react-router-dom"; +import { useDeleteDataset } from '../../api/Datasets/datasets'; +import { useState } from 'react'; +import DatasetDetailModal from '../../components/Datasets/DatasetDetails/DatasetDetailModal'; +import { DatasetResponse } from '../../api/Datasets/response'; +import { getFilesURL } from '../Evaluator/util'; +import { Pages } from "../../types"; + +const { Text } = Typography; + +const ButtonGroup = styled(Flex)` + margin-top: 15px !important; +` + +interface DatasetActionsProps { + dataset: Dataset; + refetch: () => void; + setToggleDatasetExportModal: (toggle: boolean) => void; +} + + +const DatasetActions: React.FC = ({ dataset, refetch, setToggleDatasetExportModal }) => { + const [showModal, setShowModal] = useState(false); + const deleteDatasetReq = useDeleteDataset(); + + const deleteConfirmWarningModal = (row: Dataset) => { + return Modal.warning({ + title: 'Remove Dataset', + closable: true, + content: ( + <> + + {`Are you sure you want to remove this dataset`} {row.display_name}? + + + ), + icon: undefined, + footer: ( + + + + + ), + maskClosable: true, + width: "20%" + }) + } + + async function handleDeleteEvaluationConfirm() { + await deleteDatasetReq.triggerDelete(dataset?.generate_file_name, `file_path=${dataset?.local_export_path}`); + // await datasetHistoryAPI.triggerGet(); + refetch(); + } + + const menuActions: MenuProps['items'] = [ + { + key: '1', + label: + + View Dataset Details + , + icon: + }, + { + key: '2', + label: ( + + View in Preview + + ), + icon: , + }, + { + key: '3', + label: ( + + Generate Dataset + + ), + icon: , + }, + { + key: '4', + label: ( + + Evaluate Dataset + + ), + icon: , + }, + { + key: '5', + label: ( + + Export Dataset + + ), + onClick: () => setToggleDatasetExportModal(true), + icon: + }, + { + key: '6', + label: ( + deleteConfirmWarningModal(dataset)}>Remove Dataset + ), + icon: + } + ]; + + return ( + <> + + + + + + + + {showModal && } + + + ) +} + +export default DatasetActions; \ No newline at end of file diff --git a/app/client/src/pages/Datasets/DatasetsPage.tsx b/app/client/src/pages/Datasets/DatasetsPage.tsx new file mode 100644 index 00000000..b36c85d7 --- /dev/null +++ b/app/client/src/pages/Datasets/DatasetsPage.tsx @@ -0,0 +1,213 @@ +import throttle from 'lodash/throttle'; +import React, { SyntheticEvent, useEffect } from 'react'; +import { Col, Flex, Input, Layout, Row, Table, TableProps, Tooltip, notification } from 'antd'; +import styled from 'styled-components'; +import Paragraph from 'antd/es/typography/Paragraph'; +import { useDatasets } from '../Home/hooks'; +import { ExportResult } from '../../components/Export/ExportModal'; +import { SearchProps } from 'antd/es/input'; +import Loading from '../Evaluator/Loading'; +import { Dataset } from '../Evaluator/types'; +import { JOB_EXECUTION_TOTAL_COUNT_THRESHOLD, TRANSLATIONS } from '../../constants'; +import DatasetExportModal, { ExportResult } from '../../components/Export/ExportModal'; +import DateTime from '../../components/DateTime/DateTime'; +import DatasetActions from './DatasetActions'; +import { sortItemsByKey } from '../../utils/sortutils'; + +import { JobStatus } from '../../types'; +import JobStatusIcon from '../../components/JobStatus/jobStatusIcon'; +import StyledTitle from '../Evaluator/StyledTitle'; + +const { Content } = Layout; +const { Search } = Input; + +const StyledContent = styled(Content)` + padding: 24px; + background-color: #f5f7f8; +`; + +const Container = styled.div` + background-color: #ffffff; + padding: 1rem; + overflow-x: auto; +`; + +const StyledTable = styled(Table)` + font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; + color: #5a656d; + .ant-table-thead > tr > th { + color: #5a656d; + border-bottom: 1px solid #eaebec; + font-weight: 500; + text-align: left; + // background: #ffffff; + border-bottom: 1px solid #eaebec; + transition: background 0.3s ease; + } + .ant-table-row > td.ant-table-cell { + padding: 8px; + padding-left: 16px; + font-size: 13px; + font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; + color: #5a656d; + .ant-typography { + font-size: 13px; + font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; + } + } +`; + +const StyledParagraph = styled(Paragraph)` + font-size: 13px; + font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; + color: #5a656d; +`; + +const DatasetsPage: React.FC = () => { + const { data, isLoading, isError, refetch, setSearchQuery, pagination } = useDatasets(); + const [notificationInstance, notificationContextHolder] = notification.useNotification(); + const [exportResult, setExportResult] = React.useState(); + const [toggleDatasetExportModal, setToggleDatasetExportModal] = React.useState(false); + const [datasetDetails, setDatasetDetails] = React.useState({} as Dataset); + + useEffect(() => { + if (isError) { + notification.error({ + message: 'Error', + description: 'An error occurred while fetching datasets' + }); + } + }, [isError]); + + useEffect(() => { + if (exportResult?.successMessage) { + notificationInstance.success({ + message: `Dataset Exported to Huggingface`, + description: "Dataset has been successfully exported." + }); + } + if (exportResult?.failedMessage) { + notificationInstance.error({ + message: "Error Exporting Dataset", + description: "There was an error exporting the dataset. Please try again." + }); + } + }, [exportResult, notificationInstance]) + + const onSearch: SearchProps['onSearch'] = (value: unknown) => { + throttle((value: string) => setSearchQuery(value), 500)(value); + } + + const onChange = (event: SyntheticEvent) => { + const value = (event.target as HTMLInputElement)?.value; + throttle((value: string) => setSearchQuery(value), 500)(value); + } + + const columns: TableProps['columns'] = [ + { + key: 'job_status', + title: 'Status', + dataIndex: 'job_status', + width: 80, + sorter: sortItemsByKey('job_status'), + render: (status: JobStatus) => + + + }, + { + key: 'display_name', + title: 'Display Name', + dataIndex: 'display_name', + width: 140, + sorter: sortItemsByKey('display_name') + }, { + key: 'generate_file_name', + title: 'Dataset Name', + dataIndex: 'generate_file_name', + width: 250, + sorter: sortItemsByKey('generate_file_name'), + render: (generate_file_name) => {generate_file_name} + }, { + key: 'model_id', + title: 'Model', + dataIndex: 'model_id', + width: 250, + sorter: sortItemsByKey('model_id'), + render: (modelId) => {modelId} + }, { + key: 'num_questions', + title: 'Questions Per Topic', + dataIndex: 'num_questions', + width: 120, + align: 'center', + sorter: sortItemsByKey('num_questions') + }, + { + key: 'total_count', + title: 'Total Count', + dataIndex: 'total_count', + width: 80, + align: 'center', + sorter: sortItemsByKey('total_count') + }, { + key: 'use_case', + title: 'Use Case', + dataIndex: 'use_case', + width: 120, + sorter: sortItemsByKey('use_case'), + render: (useCase) => TRANSLATIONS[useCase] + }, { + key: 'timestamp', + title: 'Creation Time', + dataIndex: 'timestamp', + defaultSortOrder: 'descend', + width: 120, + sorter: sortItemsByKey('timestamp'), + render: (timestamp) => <>{timestamp == null ? 'N/A' : } + }, { + key: '7', + title: 'Actions', + width: 100, + render: (row: Dataset) => ( + + ) + }, + ]; + return ( + + + {'Datasets'} + + + + + + + {isLoading && } + `${row?.display_name}_${row?.generate_file_name}`} + tableLayout="fixed" + pagination={pagination} + columns={columns} + dataSource={data?.data || [] as Dataset[]} + onRow={(row: Dataset) => + ({ + onClick: () => { + setDatasetDetails(row); + } + })} + /> + + {notificationContextHolder} + + + + + ); +}; + +export default DatasetsPage; \ No newline at end of file diff --git a/app/client/src/pages/Datasets/hooks.ts b/app/client/src/pages/Datasets/hooks.ts new file mode 100644 index 00000000..e69de29b diff --git a/app/client/src/pages/Evaluations/EvaluateActions.tsx b/app/client/src/pages/Evaluations/EvaluateActions.tsx new file mode 100644 index 00000000..0aeb1bd4 --- /dev/null +++ b/app/client/src/pages/Evaluations/EvaluateActions.tsx @@ -0,0 +1,124 @@ +import { useState } from "react"; +import { Dropdown, Flex, Space, MenuProps, Typography, Modal, Button } from "antd"; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import FindInPageIcon from '@mui/icons-material/FindInPage'; +import DeleteIcon from '@mui/icons-material/Delete'; +import { FolderViewOutlined, ThunderboltOutlined } from '@ant-design/icons'; +import { Link } from "react-router-dom"; +import EvaluationDetailModal from "../../components/Evaluations/EvaluationDetails/EvaluationDetailModal"; +import { Evaluation } from "./types"; +import styled from "styled-components"; +import { useDeleteEvaluation } from "../../api/Evaluations/evaluations"; +import { Pages } from "../../types"; +import { getFilesURL } from "../Evaluator/util"; + +const { Text } = Typography; + +interface Props { + evaluation: Evaluation; + refetch: () => void; +} + +const ModalButtonGroup = styled(Flex)` + margin-top: 15px !important; +`; + +const EvaluationActions: React.FC = ({ evaluation, refetch }) => { + const [showModal, setShowModal] = useState(false); + const deleteEvaluationReq = useDeleteEvaluation(); + + async function handleDeleteEvaluationConfirm() { + await deleteEvaluationReq.triggerDelete(evaluation.evaluate_file_name, `file_path=${evaluation.local_export_path}`); + refetch(); + } + + const deleteConfirmWarningModal = (row: Evaluation) => { + return Modal.warning({ + title: 'Remove Evaluation', + closable: true, + content: ( + <> + + {`Are you sure you want to remove this evaluation`} {row.display_name}? + + + ), + icon: undefined, + footer: ( + + + + + ), + maskClosable: true, + width: "20%" + }) + } + + const menuActions: MenuProps['items'] = [ + { + key: '1', + label: ( + + View Evaluation Details + + ), + icon: + }, + { + key: '2', + label: ( + + View in Preview + + ), + icon: , + }, + { + key: '3', + label: ( + + Re-evaluate + + ), + icon: , + }, + { + key: '4', + label: ( + deleteConfirmWarningModal(evaluation)}>Remove Evaluation + ), + icon: + } + ]; + + return ( + <> + + + + + + + + {showModal && + } + + ); +} + +export default EvaluationActions; \ No newline at end of file diff --git a/app/client/src/pages/Evaluations/EvaluationsPage.tsx b/app/client/src/pages/Evaluations/EvaluationsPage.tsx new file mode 100644 index 00000000..68db61cc --- /dev/null +++ b/app/client/src/pages/Evaluations/EvaluationsPage.tsx @@ -0,0 +1,170 @@ +import throttle from "lodash/throttle"; +import { SyntheticEvent, useEffect } from "react"; +import { Badge, Col, Flex, Input, Layout, notification, Row, Table, TableProps } from "antd"; +import styled from "styled-components"; +import Paragraph from 'antd/es/typography/Paragraph'; +import { JOB_EXECUTION_TOTAL_COUNT_THRESHOLD, TRANSLATIONS } from '../../constants'; +import { useEvaluations } from "../Home/hooks"; +import { Evaluation } from "../Home/types"; +import { sortItemsByKey } from "../../utils/sortutils"; +import Loading from "../Evaluator/Loading"; + +import { SearchProps } from "antd/es/input"; +import DateTime from "../../components/DateTime/DateTime"; +import EvaluateActions from "./EvaluateActions"; +import { getColorCode } from "../Evaluator/util"; +import { JobStatus } from "../../types"; +import JobStatusIcon from "../../components/JobStatus/jobStatusIcon"; +import StyledTitle from "../Evaluator/StyledTitle"; + + +const { Content } = Layout; +const { Search } = Input; + +const StyledContent = styled(Content)` + padding: 24px; + background-color: #f5f7f8; +`; + +const Container = styled.div` + background-color: #ffffff; + padding: 1rem; + overflow-x: auto; +`; + +const StyledTable = styled(Table)` + .ant-table-thead > tr > th { + color: #5a656d; + border-bottom: 1px solid #eaebec; + font-weight: 500; + text-align: left; + // background: #ffffff; + border-bottom: 1px solid #eaebec; + transition: background 0.3s ease; + } + .ant-table-row > td.ant-table-cell { + padding: 8px; + padding-left: 16px; + font-size: 14px; + font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; + color: #5a656d; + .ant-typography { + font-size: 13px; + font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; + } + } +`; + +const StyledParagraph = styled(Paragraph)` + font-size: 13px; + font-family: Roboto, -apple-system, 'Segoe UI', sans-serif; + color: #5a656d; +`; + + +const EvaluationsPage: React.FC = () => { + const { data, isLoading, isError, refetch, setSearchQuery, pagination } = useEvaluations(); + + useEffect(() => { + if (isError) { + notification.error({ + message: 'Error', + description: 'An error occurred while fetching evaluations' + }); + } + }, [isError]); + + const onSearch: SearchProps['onSearch'] = (value: unknown) => { + throttle((value: string) => setSearchQuery(value), 500)(value); + } + + const onChange = (event: SyntheticEvent) => { + const value = (event.target as HTMLInputElement)?.value; + throttle((value: string) => setSearchQuery(value), 500)(value); + } + + const columns: TableProps['columns'] = [ + { + key: 'job_status', + title: 'Status', + dataIndex: 'job_status', + width: 80, + sorter: sortItemsByKey('job_status'), + render: (status: JobStatus) => + + + }, + { + key: 'display_name', + title: 'Display Name', + dataIndex: 'display_name', + width: 150, + sorter: sortItemsByKey('display_name'), + }, { + key: 'model_id', + title: 'Model ID', + dataIndex: 'model_id', + width: 150, + sorter: sortItemsByKey('model_id'), + }, { + key: 'average_score', + title: 'Average Score', + dataIndex: 'average_score', + width: 80, + render: (average_score) => , + sorter: sortItemsByKey('average_score'), + },{ + key: 'use_case', + title: 'Use Case', + dataIndex: 'use_case', + width: 180, + sorter: sortItemsByKey('use_case'), + render: (useCase) => {TRANSLATIONS[useCase]} + }, { + key: 'timestamp', + title: 'Create Time', + dataIndex: 'timestamp', + width: 140, + sorter: sortItemsByKey('timestamp'), + render: (timestamp) => + + }, { + key: 'action', + title: 'Actions', + width: 100, + render: (row: Evaluation) => + + + }, + ]; + + + return ( + + + {'Evaluations'} + + + + + + + {isLoading && } + `${row?.display_name}_${row?.evaluate_file_name}`} + tableLayout="fixed" + pagination={pagination} + columns={columns} + dataSource={data?.data || [] as Evaluation[]} + /> + + + + ); +} + +export default EvaluationsPage; \ No newline at end of file diff --git a/app/client/src/pages/Exports/ExportsPage.tsx b/app/client/src/pages/Exports/ExportsPage.tsx new file mode 100644 index 00000000..b82b9a5f --- /dev/null +++ b/app/client/src/pages/Exports/ExportsPage.tsx @@ -0,0 +1,36 @@ +import { useEffect } from "react"; +import { Layout } from "antd"; +import styled from "styled-components"; +import ExportsTab from "../Home/ExportsTab"; +import StyledTitle from "../Evaluator/StyledTitle"; + +const { Content } = Layout; + +const StyledContent = styled(Content)` + padding: 24px; + background-color: #f5f7f8; +`; + +const Container = styled.div` + background-color: #ffffff; + padding: 1rem; + overflow-x: auto; +`; + + + +const ExportsPage: React.FC = () => { + return ( + + + + {'Exports'} + + + + + + ); +} + +export default ExportsPage; \ No newline at end of file diff --git a/app/client/src/pages/Home/DatasetsTab.tsx b/app/client/src/pages/Home/DatasetsTab.tsx index ab78872f..d73ca095 100644 --- a/app/client/src/pages/Home/DatasetsTab.tsx +++ b/app/client/src/pages/Home/DatasetsTab.tsx @@ -21,6 +21,8 @@ const { Search } = Input; const Container = styled.div` background-color: #ffffff; padding: 1rem; + padding-left: 0; + padding-right: 0; overflow-x: auto; `; @@ -55,7 +57,11 @@ const StyledParagraph = styled(Paragraph)` color: #5a656d; `; -const DatasetsTab: React.FC = () => { +interface Props { + hideSearch?: boolean; +} + +const DatasetsTab: React.FC = ({ hideSearch = false }) => { const { data, isLoading, isError, refetch, setSearchQuery, pagination } = useDatasets(); const [notificationInstance, notificationContextHolder] = notification.useNotification(); const [exportResult, setExportResult] = React.useState(); @@ -168,7 +174,7 @@ const DatasetsTab: React.FC = () => { return ( - + {!hideSearch && { onChange={onChange} style={{ width: 350 }} /> - + } {isLoading && } `${row?.display_name}_${row?.generate_file_name}`} tableLayout="fixed" - pagination={pagination} + // pagination={pagination} columns={columns} dataSource={data?.data || [] as Dataset[]} onRow={(row: Dataset) => diff --git a/app/client/src/pages/Home/EvaluateButton.tsx b/app/client/src/pages/Home/EvaluateButton.tsx deleted file mode 100644 index 83225e82..00000000 --- a/app/client/src/pages/Home/EvaluateButton.tsx +++ /dev/null @@ -1,85 +0,0 @@ -import { Button, Form, Modal, Select } from "antd"; -import { useNavigate } from 'react-router-dom'; -import ArrowRightIcon from '../../assets/ic-arrow-right.svg'; -import { useEffect, useState } from "react"; -import { useDatasets } from "./hooks"; -import Loading from "../Evaluator/Loading"; -import { isEmpty } from "lodash"; -import { Dataset } from "../Evaluator/types"; -import { Pages } from "../../types"; - - -const EvaluateButton: React.FC = () => { - const [form] = Form.useForm(); - const navigate = useNavigate(); - const [showModal, setShowModal] = useState(false); - const [datasets, setDatasets] = useState([]); - const {data, isLoading} = useDatasets(); - - useEffect(() => { - if(!isEmpty(data?.data)) { - setDatasets(data?.data); - } - }, [data]); - - const initialValues = { - dataset_name: null - } - - const onClose = () => setShowModal(false); - - const onSubmit = async () => { - try { - await form.validateFields(); - const values = form.getFieldsValue(); - const dataset = datasets.find((dataset: Dataset) => dataset.display_name === values.dataset_name); - navigate(`/${Pages.EVALUATOR}/create/${dataset?.generate_file_name}`); - } catch (e) { - console.error(e); - } - } - - const options = datasets.map((dataset: unknown) => ({ - value: dataset.display_name, - label: dataset.display_name, - key: `${dataset?.display_name}-${dataset?.generate_file_name}` - })); - - return ( - <> - - {showModal && - - {isLoading && } -
- - - + +
+ + )} + + ); +} + +export default EvaluateSection; \ No newline at end of file diff --git a/app/client/src/pages/Home/EvaluationsTab.tsx b/app/client/src/pages/Home/EvaluationsTab.tsx index 16f21edc..bcd3711a 100644 --- a/app/client/src/pages/Home/EvaluationsTab.tsx +++ b/app/client/src/pages/Home/EvaluationsTab.tsx @@ -21,6 +21,8 @@ const { Search } = Input; const Container = styled.div` background-color: #ffffff; padding: 1rem; + padding-left: 0; + padding-right: 0; overflow-x: auto; `; @@ -53,8 +55,12 @@ const StyledParagraph = styled(Paragraph)` color: #5a656d; `; +interface Props { + hideSearch?: boolean; +} + -const EvaluationsTab: React.FC = () => { +const EvaluationsTab: React.FC = ({ hideSearch }) => { const { data, isLoading, isError, refetch, setSearchQuery, pagination } = useEvaluations(); useEffect(() => { @@ -132,7 +138,7 @@ const EvaluationsTab: React.FC = () => { return ( - + {!hideSearch && { onChange={onChange} style={{ width: 350 }} /> - + } {isLoading && } `${row?.display_name}_${row?.evaluate_file_name}`} tableLayout="fixed" - pagination={pagination} + // pagination={pagination} columns={columns} dataSource={data?.data || [] as Evaluation[]} /> diff --git a/app/client/src/pages/Home/ExportsTab.tsx b/app/client/src/pages/Home/ExportsTab.tsx index a81f870f..f0c458c8 100644 --- a/app/client/src/pages/Home/ExportsTab.tsx +++ b/app/client/src/pages/Home/ExportsTab.tsx @@ -16,6 +16,8 @@ const { Text, Link, Paragraph } = Typography; const Container = styled.div` background-color: #ffffff; padding: 1rem; + padding-left: 0; + padding-right: 0; `; const StyledTable = styled(Table)` @@ -51,6 +53,8 @@ const StyledParagraph = styled(Paragraph)` export type ExportsTabProps = { refetchOnRender: boolean; + hideSearch?: boolean; + hidePagination?: boolean; }; const columns: TableProps['columns'] = [ @@ -108,7 +112,7 @@ const columns: TableProps['columns'] = [ } ]; -const ExportsTab: React.FC = ({ refetchOnRender }) => { +const ExportsTab: React.FC = ({ refetchOnRender, hideSearch, hidePagination }) => { const [pagination, setPagination] = useState({ page: 1, pageSize: 20 }); const { isLoading, data, refetch } = useGetExportJobs(pagination.page, pagination.pageSize); const [searchTerm, setSearchTerm] = useState(''); @@ -137,7 +141,7 @@ const ExportsTab: React.FC = ({ refetchOnRender }) => { return ( - + {!hideSearch && = ({ refetchOnRender }) => { }} style={{ width: 350 }} /> - + } row.id} columns={columns} tableLayout="fixed" - pagination={{ + pagination={hidePagination ? false : { current: pagination.page, pageSize: pagination.pageSize, total: data?.pagination?.total || 0, diff --git a/app/client/src/pages/Home/HomePage.tsx b/app/client/src/pages/Home/HomePage.tsx index d92197a4..d1768803 100644 --- a/app/client/src/pages/Home/HomePage.tsx +++ b/app/client/src/pages/Home/HomePage.tsx @@ -1,15 +1,15 @@ import React, { useState } from 'react'; import styled from 'styled-components'; -import { Button, Col, Flex, Layout, Row, Tabs } from 'antd' +import { Card, Col, Flex, Layout, Row, Tabs } from 'antd' import type { TabsProps } from 'antd'; import DatasetsTab from './DatasetsTab'; import EvaluationsTab from './EvaluationsTab'; -import DatasetIcon from '../../assets/ic-datasets.svg'; -import ArrowRightIcon from '../../assets/ic-arrow-right.svg'; -import EvaluateIcon from '../../assets/ic-evaluations.svg'; +import DatasetIcon from '../../assets/ic-brand-alternative-data.svg'; import DataAugmentationIcon from '../../assets/ic-data-augmentation.svg'; -import EvaluateButton from './EvaluateButton'; import ExportsTab from './ExportsTab'; +import TemplatesSection from './TemplatesSection'; +import { useNavigate } from 'react-router-dom'; +import EvaluateSection from './EvaluateSection'; const { Content } = Layout; @@ -19,33 +19,35 @@ const StyledContent = styled(Content)` background-color: #f5f7f8; `; -const HeaderSection = styled.div` +export const HeaderSection = styled.div` display: flex; margin-bottom: 1rem; height: 100px; width: 50%; padding: 16px; background-color: #ffffff; + cursor: pointer; .left-section { width: 66px; - height: 66px; + height: 46px; flex-grow: 0; margin: 0 8px 9px 0; padding: 14.4px 14.4px 14.4px 14.4px; - background-color: #e5ffe5; + background-color: #ffffff; } .middle-section { display: flex; flex-direction: column; justify-content: center; margin-left: 8px; + margin-top: 12px; width: 70%; .section-title { width: 186px; height: 24px; flex-grow: 0; font-size: 16px; - font-weight: 500; + font-weight: normal; font-stretch: normal; font-style: normal; line-height: 1.5; @@ -53,13 +55,25 @@ const HeaderSection = styled.div` text-align: left; color: #1b2329; } - } - .right-section { + .section-description { + align-self: stretch; + flex-grow: 1; + font-size: 12px; + font-weight: normal; + font-stretch: normal; + font-style: normal; + line-height: 1.33; + letter-spacing: normal; + text-align: left; + color: #1b2329; + } + } + .right-section { display: flex; flex-direction: column-reverse; } .evaluate-icon { - background-color: #fff4cd; + background-color: #ffffff; } `; @@ -70,23 +84,24 @@ export enum ViewType { } const HomePage: React.FC = () => { + const navigate = useNavigate(); const [tabViewType, setTabViewType] = useState(ViewType.DATASETS); const items: TabsProps['items'] = [ { key: ViewType.DATASETS, label: 'Datasets', - children: , + children: , }, { key: ViewType.EVALUATIONS, label: 'Evaluations', - children: , + children: , }, { key: ViewType.EXPORTS, label: 'Exports', - children: , + children: , } ]; @@ -98,69 +113,47 @@ const HomePage: React.FC = () => { - + navigate('/data-generator')}>
Datasets
Create Datasets
-

Generate synthetic datasets for training models

-
-
-
-
- + Generate synthetic datasets for training models
- + + navigate('/data-augmentation')}>
Datasets
Data Augmentation
-

Generate multi-dimension datasets using LLM custom prompts

-
-
-
-
- -
-
- - -
- Datasets -
-
-
Evaluate
-
-

Evaluate generated datasets for fine tuning LLMs

-
-
-
-
- + Generate multi-dimension datasets using LLM custom prompts
+ - + + + + + + + + ); diff --git a/app/client/src/pages/Home/TemplateCard.tsx b/app/client/src/pages/Home/TemplateCard.tsx new file mode 100644 index 00000000..112a68e2 --- /dev/null +++ b/app/client/src/pages/Home/TemplateCard.tsx @@ -0,0 +1,213 @@ +import React from 'react'; +import styled from "styled-components"; +import { Template } from './types'; +import { Popover, Space, Tag } from 'antd'; +import ArrowRightIcon from '../../assets/ic-arrow-right-light.svg'; +import { Pages } from '../../types'; +import { useNavigate } from 'react-router-dom'; +import sample from 'lodash/sample'; +import { getTemplateTagColors, TemplateTagThemes } from './constans'; + + +interface Props { + template: Template; +} + +const StyledCard = styled.div` + background-color: #ffffff; + display: flex; + flex-direction: column; + overflow-x: auto; + height: 200px; + width: 300px; + align-self: stretch; + flex-grow: 0; + justify-content: flex-start; + align-items: stretch; + gap: 8px; + padding: 16px 24px; + border-radius: 4px; + border: 1px solid #d6d8db; + cursor: pointer; + +`; + +const TopSection = styled.div` + display: flex; + flex-direction: column; + justify-content: center; + align-items: flex-start; + flex: 1; + margin-bottom: 1rem; +`; + +const StyledTitle = styled.div` + height: 24px; + flex-grow: 0; + font-size: 16px; + font-weight: normal; + font-stretch: normal; + font-style: normal; + line-height: 1.5; + letter-spacing: normal; + text-align: left; + color: rgba(0, 0, 0, 0.85); +`; + +const StyledDescription = styled.div` + height: 44px; + align-self: stretch; + flex-grow: 0; + font-size: 14px; + font-weight: normal; + font-stretch: normal; + font-style: normal; + line-height: 1.57; + letter-spacing: normal; + text-align: left; + color: rgba(0, 0, 0, 0.45); + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; + display: -webkit-box; + -webkit-line-clamp: 2; /* Number of lines to show */ + -webkit-box-orient: vertical; + text-overflow: ellipsis; +`; + + + +const BottomSection = styled.div` + flex: 1; + display: flex; + align-items: center; + height: 32px; + flex-grow: 1; + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: center; + padding: 0; + .text { + width: 78px; + height: 24px; + flex-grow: 0; + font-size: 14px; + font-weight: normal; + font-stretch: normal; + font-style: normal; + line-height: 1.57; + letter-spacing: normal; + text-align: left; + color: rgba(0, 0, 0, 0.88); + } + .icon { + width: 24px; + height: 24px; + flex-grow: 0; + display: flex; + justify-content: center; + align-items: center; + border-radius: 50%; + cursor: pointer; + color: #000000e1; + } +`; + +const TagsContainer = styled.div` + min-height: 30px; + display: block; + margin-bottom: 4px; + margin-top: 4px; + .ant-tag { + max-width: 150px; + } + .tag-title { + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } +`; + +const StyledTag = styled(Tag)` + color: ${props => props.theme.color}; + background-color: ${props => props.theme.backgroundColor}; + border: 1px solid ${props => props.theme.borderColor}; +`; + + +const TemplateCard: React.FC = ({ template }) => { + const navigate = useNavigate(); + const hasTags = template.tag !== null && Array.isArray(template.tag); + const tags = !hasTags ? [] : template.tag.slice(0, 1); + const moreTags = !hasTags ? [] : template.tag.slice(1); + + const getTag = (tag: string) => { + const theme = sample(TemplateTagThemes); + const { color, backgroundColor, borderColor } = getTemplateTagColors(theme as string); + + return ( + +
+ {tag} +
+ + ) + } + + + return ( + navigate(`/${Pages.GENERATOR}/${template.id}`)}> + + {template.name} + + {template.description} + + + + + {tags.map((tag: string) => ( + getTag(tag) + ))} + {moreTags.length > 0 && ( + + {moreTags.map((tag: string) => ( + getTag(tag) + ))} + + } + trigger="hover" + > + +
{`+${moreTags.length}`}
+ + + )} + + + + Get Started +
+ Get Started +
+ + + ) +} + +export default TemplateCard; \ No newline at end of file diff --git a/app/client/src/pages/Home/TemplatesSection.tsx b/app/client/src/pages/Home/TemplatesSection.tsx new file mode 100644 index 00000000..69379513 --- /dev/null +++ b/app/client/src/pages/Home/TemplatesSection.tsx @@ -0,0 +1,58 @@ +import get from 'lodash/get'; +import { Card } from 'antd'; +import React from 'react'; +import styled from "styled-components"; +import { useGetUseCases } from '../DataGenerator/hooks'; +import Loading from '../Evaluator/Loading'; +import { Pages } from "../../types"; +import { Template } from './types'; +import TemplateCard from './TemplateCard'; + + +const Container = styled.div` + background-color: #ffffff; + padding: 1rem; + overflow-x: auto; +`; + +const StyledContainer = styled.div` + display: flex; + flex-wrap: wrap; + gap: 16px; + padding: 16px; + width: 100%; + // justify-content: center; + // align-items: center; +`; + + +const TemplatesSection: React.FC = () => { + const useCasesReq = useGetUseCases(); + console.log("useCasesReq", useCasesReq); + if (useCasesReq.isLoading) { + return ; + } + + const useCases: Template[] = get(useCasesReq, 'data.usecases', []); + console.log("useCases", useCases); + + + return ( + + {useCasesReq.isLoading && } + {useCasesReq.isError &&
Error loading templates
} + + {useCases.map((useCase: Template) => ( + ) + )} + + + + + ) +} + +export default TemplatesSection; \ No newline at end of file diff --git a/app/client/src/pages/Home/constans.ts b/app/client/src/pages/Home/constans.ts new file mode 100644 index 00000000..030d19c1 --- /dev/null +++ b/app/client/src/pages/Home/constans.ts @@ -0,0 +1,43 @@ + +export const TemplateTagThemes = [ 'green', 'blue', 'yellow' ]; + +export enum TemplateColors { + DARK_GREEN = '#0a5f0a', + LIGHT_GREEN = '#e5ffe5', + BORDER_GREEN = '#acfbac', + DARK_BLUE = '#004379', + LIGHT_BLUE = '#edf7ff', + BORDER_BLUE = '#90ceff', + LIGHT_YELLOW = '#fff4cd', + DARK_YELLOW = '#6e5600', + BORDER_YELLOW = '#fce079' +} + +export const getTemplateTagColors = (theme: string) => { + switch (theme) { + case 'green': + return { + color: TemplateColors.DARK_GREEN, + backgroundColor: TemplateColors.LIGHT_GREEN, + borderColor: TemplateColors.BORDER_GREEN + }; + case 'blue': + return { + color: TemplateColors.DARK_BLUE, + backgroundColor: TemplateColors.LIGHT_BLUE, + borderColor: TemplateColors.BORDER_BLUE + }; + case 'yellow': + return { + color: TemplateColors.DARK_YELLOW, + backgroundColor: TemplateColors.LIGHT_YELLOW, + borderColor: TemplateColors.BORDER_YELLOW + }; + default: + return { + color: TemplateColors.DARK_GREEN, + backgroundColor: TemplateColors.LIGHT_GREEN, + borderColor: TemplateColors.BORDER_GREEN + }; + } +}; \ No newline at end of file diff --git a/app/client/src/pages/Home/hooks.ts b/app/client/src/pages/Home/hooks.ts index aa40d5e6..a0061dc9 100644 --- a/app/client/src/pages/Home/hooks.ts +++ b/app/client/src/pages/Home/hooks.ts @@ -35,7 +35,7 @@ const fetchExports = async (page = 1, pageSize = 10) => { export const useDatasets = () => { const [searchQuery, setSearchQuery] = useState(null); - const [pagination, setPagination] = useState({ page: 1, pageSize: 10 }); + const [pagination, setPagination] = useState({ page: 1, pageSize: 5 }); const { data, isLoading, isError, refetch } = useQuery>( { @@ -65,6 +65,9 @@ export const useDatasets = () => { data: filtered, }; } + if (filteredData && filteredData.data.length !== 0 && filteredData.data.length > 5) { + filteredData.data = filteredData.data.slice(0, 5); + } return { data: filteredData, @@ -116,6 +119,10 @@ export const useEvaluations = () => { data: filtered, }; } + + if (filteredData && filteredData.data.length !== 0 && filteredData.data.length > 5) { + filteredData.data = filteredData.data.slice(0, 5); + } return { data: filteredData, @@ -170,6 +177,10 @@ export const useExports = () => { }; } + + if (filteredData && filteredData.data.length !== 0 && filteredData.data.length > 5) { + filteredData.data = filteredData.data.slice(0, 5); + } return { data: filteredData, diff --git a/app/client/src/pages/Home/types.ts b/app/client/src/pages/Home/types.ts index d5c31b2b..62c5d18f 100644 --- a/app/client/src/pages/Home/types.ts +++ b/app/client/src/pages/Home/types.ts @@ -31,4 +31,11 @@ export interface DatasetDetails { export interface DatasetGeneration { [key: string]: string; +} + +export interface Template { + id: string; + name: string; + tag: string[]; + description: string; } \ No newline at end of file diff --git a/app/client/src/routes.tsx b/app/client/src/routes.tsx index 59e20a38..9a0e4cc2 100644 --- a/app/client/src/routes.tsx +++ b/app/client/src/routes.tsx @@ -9,6 +9,9 @@ import DatasetDetailsPage from "./pages/DatasetDetails/DatasetDetailsPage"; import WelcomePage from "./pages/Home/WelcomePage"; import ErrorPage from "./pages/ErrorPage"; import EvaluationDetailsPage from "./pages/EvaluationDetails/EvaluationDetailsPage"; +import DatasetsPage from "./pages/Datasets/DatasetsPage"; +import EvaluationsPage from "./pages/Evaluations/EvaluationsPage"; +import ExportsPage from "./pages/Exports/ExportsPage"; //import TelemetryDashboard from "./components/TelemetryDashboard"; @@ -39,12 +42,36 @@ const router = createBrowserRouter([ errorElement: , loader: async () => null }, + { + path: `${Pages.GENERATOR}/:template_name`, + element: , + errorElement: , + loader: async () => null + }, { path: Pages.DATA_AUGMENTATION, element: , errorElement: , loader: async () => null }, + { + path: Pages.DATASETS, + element: , + errorElement: , + loader: async () => null + }, + { + path: Pages.EVALUATIONS, + element: , + errorElement: , + loader: async () => null + }, + { + path: Pages.EXPORTS, + element: , + errorElement: , + loader: async () => null + }, { path: `${Pages.REGENERATE}/:generate_file_name`, element: , diff --git a/app/client/src/types.ts b/app/client/src/types.ts index 33b420fe..900b81ab 100644 --- a/app/client/src/types.ts +++ b/app/client/src/types.ts @@ -6,6 +6,8 @@ export enum Pages { HISTORY = 'history', HOME = 'home', DATASETS = 'datasets', + EVALUATIONS = 'evaluations', + EXPORTS = 'exports', WELCOME = 'welcome', FEEDBACK = 'feedback', UPGRADE = 'upgrade' From 98241b8f3126e38a73219dfac1888fc282ce1bc3 Mon Sep 17 00:00:00 2001 From: Khauneesh Saigal Date: Thu, 24 Jul 2025 19:45:52 +0530 Subject: [PATCH 19/26] modified build script to directly install from pip rather than astral.sh --- build/shell_scripts/build_client.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build/shell_scripts/build_client.sh b/build/shell_scripts/build_client.sh index dc289f3d..0ea6a63f 100644 --- a/build/shell_scripts/build_client.sh +++ b/build/shell_scripts/build_client.sh @@ -5,10 +5,13 @@ set -eox pipefail export UV_HTTP_TIMEOUT=3600 # Ensure uv is installed -if ! command -v uv &> /dev/null; then - echo "Installing uv package manager..." - curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="$HOME/.cargo/bin:$PATH" +set +e +uv --version >/dev/null 2>&1 +return_code=$? +set -e +if [ $return_code -ne 0 ]; then + echo "Installing uv package manager via pip..." + python -m pip install uv fi # Setup virtual environment and dependencies From c0c9242a476bfabd06f92c5a168ec2835be434eb Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Thu, 24 Jul 2025 14:51:08 -0700 Subject: [PATCH 20/26] New Descriptions for Home Page Cards --- app/client/src/pages/Home/EvaluateSection.tsx | 6 +++--- app/client/src/pages/Home/HomePage.tsx | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/client/src/pages/Home/EvaluateSection.tsx b/app/client/src/pages/Home/EvaluateSection.tsx index 8687d9df..e4153bfa 100644 --- a/app/client/src/pages/Home/EvaluateSection.tsx +++ b/app/client/src/pages/Home/EvaluateSection.tsx @@ -58,12 +58,12 @@ const EvaluateSection: React.FC = () => { <>
- Datasets + evaluation
-
Evaluate
+
Evaluation
- Evaluate generated datasets for fine tuning LLMs + Use LLMs to score and filter your synthetic data. Keep only high-quality results.
diff --git a/app/client/src/pages/Home/HomePage.tsx b/app/client/src/pages/Home/HomePage.tsx index d1768803..7dccb7cd 100644 --- a/app/client/src/pages/Home/HomePage.tsx +++ b/app/client/src/pages/Home/HomePage.tsx @@ -118,21 +118,21 @@ const HomePage: React.FC = () => { Datasets
-
Create Datasets
+
Generation
- Generate synthetic datasets for training models + Create synthetic data from scratch using examples, documents, seed instructions and AI assisted prompts.
navigate('/data-augmentation')}>
- Datasets + augmentation
-
Data Augmentation
+
Augmentation
- Generate multi-dimension datasets using LLM custom prompts + Add synthetic rows or field to existing data to fill gaps or balance datasets such as language translations.
From 1a1a92a0ef9d6bae0ec453e6e01137da5182bf51 Mon Sep 17 00:00:00 2001 From: Khauneesh Saigal Date: Fri, 25 Jul 2025 17:43:45 +0530 Subject: [PATCH 21/26] cetificates issues handled for PVC | correct error propagation for failed requests and jobs --- .gitignore | 2 ++ app/core/model_handlers.py | 22 ++++++++++++++++++++-- app/services/synthesis_service.py | 19 ++++++++++++------- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index cf75843f..0549d46a 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ sample_200x100.csv Raw_Web_Visit_sample.csv Raw_Web_Visit_Sample.csv Raw_Web_Visit_Sample.csv +app/test_models.py +credit_card_example.json diff --git a/app/core/model_handlers.py b/app/core/model_handlers.py index 7df26ad4..92391b8c 100644 --- a/app/core/model_handlers.py +++ b/app/core/model_handlers.py @@ -316,9 +316,18 @@ def _handle_openai_request(self, prompt: str): pool=5.0 ) + # Configure httpx client with certificate verification for private cloud + if os.path.exists("/etc/ssl/certs/ca-certificates.crt"): + http_client = httpx.Client( + verify="/etc/ssl/certs/ca-certificates.crt", + timeout=timeout_config + ) + else: + http_client = httpx.Client(timeout=timeout_config) + client = OpenAI( api_key=os.getenv('OPENAI_API_KEY'), - timeout=timeout_config + http_client=http_client ) completion = client.chat.completions.create( model=self.model_id, @@ -380,10 +389,19 @@ def _handle_caii_request(self, prompt: str): pool=5.0 ) + # Configure httpx client with certificate verification for private cloud + if os.path.exists("/etc/ssl/certs/ca-certificates.crt"): + http_client = httpx.Client( + verify="/etc/ssl/certs/ca-certificates.crt", + timeout=timeout_config + ) + else: + http_client = httpx.Client(timeout=timeout_config) + client_ca = OpenAI( base_url=caii_endpoint, api_key=API_KEY, - timeout=timeout_config # Use the comprehensive timeout configuration + http_client=http_client ) completion = client_ca.chat.completions.create( diff --git a/app/services/synthesis_service.py b/app/services/synthesis_service.py index 7f893f9e..e42ac3c8 100644 --- a/app/services/synthesis_service.py +++ b/app/services/synthesis_service.py @@ -1007,19 +1007,24 @@ async def generate_freeform(self, request: SynthesisRequest, job_name=None, is_d json.dump(final_output, indent=2, fp=f) self.logger.info(f"Saved {len(final_output)} results to {file_path}") - # Check if we have any critical model errors across all topics - has_critical_model_error = any( - topic_errors and any("ModelHandlerError" in error for error in topic_errors) - for _, _, topic_errors, _ in completed_topics - ) + # Find the first critical model error message + first_critical_error = None + for _, _, topic_errors, _ in completed_topics: + if topic_errors: + for error in topic_errors: + if "ModelHandlerError" in error: + first_critical_error = error + break + if first_critical_error: + break # After saving (or if no data), check for critical errors - if has_critical_model_error: + if first_critical_error: if final_output: self.logger.info(f"Saved {len(final_output)} results before failing due to model errors") else: self.logger.info("No results to save before failing due to model errors") - raise APIError("Critical model errors encountered during generation") + raise APIError(first_critical_error) # Handle custom prompt, examples and schema custom_prompt_str = PromptHandler.get_default_custom_prompt(request.use_case, request.custom_prompt) From 6462dfb5e1e0b7d58105c812f0d28d6d1ae65abf Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Thu, 24 Jul 2025 22:49:33 -0700 Subject: [PATCH 22/26] fix: minor update to DataGenerator Configure component for consistency before PR creation --- .../src/pages/DataGenerator/Configure.tsx | 14 +++-- .../DataGenerator/CustomPromptButton.tsx | 2 +- .../src/pages/DataGenerator/Examples.tsx | 45 ++++++++++------ app/client/src/pages/DataGenerator/hooks.ts | 1 + app/client/src/pages/Home/EvaluateSection.tsx | 25 ++++++--- app/client/src/pages/Home/HomePage.tsx | 51 ++++++++++++++++--- 6 files changed, 103 insertions(+), 35 deletions(-) diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index e1494352..0bb50ce8 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -13,7 +13,6 @@ import FileSelectorButton from './FileSelectorButton'; import UseCaseSelector from './UseCaseSelector'; import { useLocation, useParams } from 'react-router-dom'; import { WizardModeType } from '../../types'; -import { get } from 'lodash'; const StepContainer = styled(Flex)` @@ -96,10 +95,13 @@ const Configure = () => { validateForm() }, [form, formData]) - // keivan + useEffect(() => { if (formData && formData?.inference_type === undefined && isEmpty(generate_file_name)) { form.setFieldValue('inference_type', ModelProviders.CAII); + setTimeout(() => { + form.setFieldValue('use_case','custom'); + }, 1000); } }, [formData]); @@ -251,18 +253,19 @@ const Configure = () => { {(formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) && - } + } {( formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || formData?.workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) && ({ @@ -307,6 +310,7 @@ const Configure = () => { label='Input Key' labelCol={labelCol} validateTrigger={['workflow_type', 'onChange']} + tooltip='Choose the key or column from your uploaded file that will be used as the input for data generation.' shouldUpdate rules={[ () => ({ @@ -330,6 +334,7 @@ const Configure = () => { name='output_key' label='Output Key' labelCol={labelCol} + tooltip='Name the value or column where the prompts will be saved. If left blank, this will default to “Prompt".' shouldUpdate > @@ -338,6 +343,7 @@ const Configure = () => { name='output_value' label='Output Value' labelCol={labelCol} + tooltip='Enter the name for the generated values corresponding to each input. If left blank, this will default to “Completion”.' shouldUpdate > diff --git a/app/client/src/pages/DataGenerator/CustomPromptButton.tsx b/app/client/src/pages/DataGenerator/CustomPromptButton.tsx index e3fe10c4..7a725129 100644 --- a/app/client/src/pages/DataGenerator/CustomPromptButton.tsx +++ b/app/client/src/pages/DataGenerator/CustomPromptButton.tsx @@ -157,7 +157,7 @@ const CustomPromptButton: React.FC = ({ model_id, inference_type, caii_en disabled={mutation.isPending} rows={15} autoSize - placeholder={'Enter instructions for a custom prompt'} + placeholder={'Generate prompt from the example data'} /> diff --git a/app/client/src/pages/DataGenerator/Examples.tsx b/app/client/src/pages/DataGenerator/Examples.tsx index 2fc519ff..d0ffe6c3 100644 --- a/app/client/src/pages/DataGenerator/Examples.tsx +++ b/app/client/src/pages/DataGenerator/Examples.tsx @@ -12,7 +12,7 @@ import PCModalContent from './PCModalContent'; import { ExampleType, File, QuestionSolution, WorkflowType } from './types'; import FileSelectorButton from './FileSelectorButton'; -import { fetchFileContent, getExampleType, useGetExamplesByUseCase } from './hooks'; +import { fetchExamplesByUseCase, fetchFileContent, getExampleType, useGetExamplesByUseCase } from './hooks'; import { useState } from 'react'; import FreeFormExampleTable from './FreeFormExampleTable'; import { L } from 'vitest/dist/chunks/reporters.DTtkbAtP.js'; @@ -54,13 +54,21 @@ const Examples: FunctionComponent = () => { const form = Form.useFormInstance(); const [records, setRecords] = useState[]>([]); const workflowType = form.getFieldValue('workflow_type'); - const { examples, isLoading: examplesLoading, refetch } = - useGetExamplesByUseCase(form.getFieldValue('use_case')); const mutation = useMutation({ mutationFn: fetchFileContent }); + const restore_mutation = useMutation({ + mutationFn: fetchExamplesByUseCase + }); + + useEffect(() => { + const useCase = form.getFieldValue('use_case'); + restore_mutation.mutate(useCase); + }, [form.getFieldValue('use_case')]); + + useEffect(() => { const example_path = form.getFieldValue('example_path'); if (!isEmpty(example_path)) { @@ -70,21 +78,28 @@ const Examples: FunctionComponent = () => { } }, [form.getFieldValue('example_path'), form.getFieldValue('workflow_type')]); - useEffect(() => { - console.log('------------------> useEffect') + useEffect(() => { if (!isEmpty(mutation.data)) { form.setFieldValue('examples', mutation.data); if (!isEqual(mutation.data, records)) { setRecords(mutation.data); } - } else if (Array.isArray(examples) && !isEqual(examples, records)) { + } + }, [mutation.data]); + + useEffect(() => { + if (!isEmpty(restore_mutation.data)) { + const examples = get(restore_mutation.data, 'examples', []); form.setFieldValue('examples', examples); setRecords(examples); } - }, [mutation.data, examples]); - + }, [restore_mutation.data]); + const onRestoreDefaults = async() => { + const useCase = form.getFieldValue('use_case'); + restore_mutation.mutate(useCase); + } const onAddFiles = (files: File[]) => { if (!isEmpty (files)) { @@ -104,16 +119,16 @@ const Examples: FunctionComponent = () => { span: 10 }; - const showEmptyState = workflowType === WorkflowType.FREE_FORM_DATA_GENERATION && + const showEmptyState = (workflowType === WorkflowType.FREE_FORM_DATA_GENERATION && isEmpty(mutation.data) && - records.length === 0; + records.length === 0) || + (form.getFieldValue('use_case') === 'custom' && + isEmpty(form.getFieldValue('examples'))); - console.log('examples', form.getFieldValue('examples')); - console.log('records', records); return ( - {examplesLoading && } + {mutation?.isPending || restore_mutation.isPending && }
@@ -149,8 +164,8 @@ const Examples: FunctionComponent = () => { diff --git a/app/client/src/pages/Home/HomePage.tsx b/app/client/src/pages/Home/HomePage.tsx index 7dccb7cd..61385f1c 100644 --- a/app/client/src/pages/Home/HomePage.tsx +++ b/app/client/src/pages/Home/HomePage.tsx @@ -1,6 +1,6 @@ import React, { useState } from 'react'; import styled from 'styled-components'; -import { Card, Col, Flex, Layout, Row, Tabs } from 'antd' +import { Button, Card, Col, Flex, Layout, Row, Tabs } from 'antd' import type { TabsProps } from 'antd'; import DatasetsTab from './DatasetsTab'; import EvaluationsTab from './EvaluationsTab'; @@ -10,6 +10,7 @@ import ExportsTab from './ExportsTab'; import TemplatesSection from './TemplatesSection'; import { useNavigate } from 'react-router-dom'; import EvaluateSection from './EvaluateSection'; +import ArrowRightIcon from '../../assets/ic-arrow-right.svg'; const { Content } = Layout; @@ -21,12 +22,23 @@ const StyledContent = styled(Content)` export const HeaderSection = styled.div` display: flex; + flex-direction: column; margin-bottom: 1rem; - height: 100px; + height: 150px; width: 50%; padding: 16px; background-color: #ffffff; cursor: pointer; + .top-section { + display: flex; + flex-direction: row; + } + .bottom-section { + display: flex; + flex-direction: row; + justify-content: flex-end; + margin-top: 8px; + } .left-section { width: 66px; height: 46px; @@ -66,6 +78,7 @@ export const HeaderSection = styled.div` letter-spacing: normal; text-align: left; color: #1b2329; + min-height: 50px; } } .right-section { @@ -114,6 +127,7 @@ const HomePage: React.FC = () => { navigate('/data-generator')}> +
Datasets
@@ -123,16 +137,37 @@ const HomePage: React.FC = () => { Create synthetic data from scratch using examples, documents, seed instructions and AI assisted prompts.
+ + +
+
+ +
+
navigate('/data-augmentation')}> -
- augmentation +
+
+ augmentation +
+
+
Augmentation
+
+ Add synthetic rows or field to existing data to fill gaps or balance datasets such as language translations. +
+
-
-
Augmentation
-
- Add synthetic rows or field to existing data to fill gaps or balance datasets such as language translations. + +
+
+
From e0375a8c24c92a8dd7c33f0a955930bfbba3d40a Mon Sep 17 00:00:00 2001 From: Khauneesh Saigal Date: Sat, 26 Jul 2025 22:16:25 +0530 Subject: [PATCH 23/26] adding completed rows on the final homepage datasets tab --- app/client/src/api/Datasets/response.ts | 1 + app/client/src/pages/Home/DatasetsTab.tsx | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/client/src/api/Datasets/response.ts b/app/client/src/api/Datasets/response.ts index f3738346..c58ab011 100644 --- a/app/client/src/api/Datasets/response.ts +++ b/app/client/src/api/Datasets/response.ts @@ -22,6 +22,7 @@ export type DatasetResponse = { job_name: string; job_status: string; inference_type: string; + completed_rows: number; // Add this line }; export type ModelParameters = { diff --git a/app/client/src/pages/Home/DatasetsTab.tsx b/app/client/src/pages/Home/DatasetsTab.tsx index d73ca095..7ce2d040 100644 --- a/app/client/src/pages/Home/DatasetsTab.tsx +++ b/app/client/src/pages/Home/DatasetsTab.tsx @@ -147,7 +147,16 @@ const DatasetsTab: React.FC = ({ hideSearch = false }) => { width: 80, align: 'center', sorter: sortItemsByKey('total_count') - }, { + }, + { + key: 'completed_rows', + title: 'Completed Rows', + dataIndex: 'completed_rows', + width: 80, + align: 'center', + sorter: sortItemsByKey('completed_rows') + }, + { key: 'use_case', title: 'Use Case', dataIndex: 'use_case', From 7331eef9aa6b4b46af298e1d63e99a5f36a9b9d3 Mon Sep 17 00:00:00 2001 From: Khauneesh Saigal Date: Mon, 28 Jul 2025 16:03:31 +0530 Subject: [PATCH 24/26] evaluate default examples bug resolved --- app/core/prompt_templates.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/core/prompt_templates.py b/app/core/prompt_templates.py index f187ae7d..7500177f 100644 --- a/app/core/prompt_templates.py +++ b/app/core/prompt_templates.py @@ -572,8 +572,7 @@ def get_freeform_eval_prompt(model_id: str, examples_str = PromptHandler.format_examples_eval(examples) elif examples == [] or examples == None: - examples_str = PromptHandler.format_examples_eval(USE_CASE_CONFIGS_EVALS[use_case].default_examples) - + examples_str = str(USE_CASE_CONFIGS_EVALS[use_case].default_examples) base_prompt = """ You are a brilliant judge on evaluating a set of data with fields and corresponding values Follow the given instructions to understand the structure of given data and evaluate it based on parameters defined for you.""" final_instruction = f"""data row: {row} From 3e1568d82a6edf68f4e09d3f52aab2ab66b54642 Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Mon, 28 Jul 2025 10:58:13 -0700 Subject: [PATCH 25/26] Fix Regenerate Issues Re-generate issues --- .../src/pages/DataGenerator/Configure.tsx | 16 ++-- .../src/pages/DataGenerator/DataGenerator.tsx | 85 +++++++++++-------- .../src/pages/DataGenerator/Examples.tsx | 18 +++- app/client/src/pages/DataGenerator/Prompt.tsx | 1 - .../pages/DataGenerator/UseCaseSelector.tsx | 1 - app/client/src/pages/DataGenerator/hooks.ts | 1 - .../src/pages/Home/TemplatesSection.tsx | 2 - 7 files changed, 73 insertions(+), 51 deletions(-) diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index 0bb50ce8..1326a1eb 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -1,8 +1,8 @@ import endsWith from 'lodash/endsWith'; import isEmpty from 'lodash/isEmpty'; import isFunction from 'lodash/isFunction'; -import { useEffect, useState } from 'react'; -import { Flex, Form, Input, Select, Typography } from 'antd'; +import { FunctionComponent, useEffect, useState } from 'react'; +import { Flex, Form, FormInstance, Input, Select, Typography } from 'antd'; import styled from 'styled-components'; import { File, WorkflowType } from './types'; import { useFetchModels } from '../../api/api'; @@ -49,7 +49,9 @@ export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [ { label: MODEL_PROVIDER_LABELS[ModelProviders.CAII], value: ModelProviders.CAII }, ]; -const Configure = () => { +const Configure: FunctionComponent = () => { + const form = Form.useFormInstance(); + const formData = Form.useWatch((values) => values, form); const location = useLocation(); const { template_name, generate_file_name } = useParams(); const [wizardModeType, setWizardModeType] = useState(getWizardModeType(location)); @@ -57,10 +59,10 @@ const Configure = () => { useEffect(() => { if (wizardModeType === WizardModeType.DATA_AUGMENTATION) { setWizardModeType(WizardModeType.DATA_AUGMENTATION); - form.setFieldValue('workflow_type', 'custom'); + form.setFieldValue('workflow_type', 'freeform'); } else { setWizardModeType(WizardModeType.DATA_GENERATION); - form.setFieldValue('workflow_type', 'freeform'); + form.setFieldValue('workflow_type', 'custom'); } }, [location, wizardModeType]); @@ -70,8 +72,8 @@ const Configure = () => { } }, [template_name]); - const form = Form.useFormInstance(); - const formData = Form.useWatch((values) => values, form); + + // let formData = Form.useWatch((values) => values, form); const { setIsStepValid } = useWizardCtx(); const { data } = useFetchModels(); const [selectedFiles, setSelectedFiles] = useState( diff --git a/app/client/src/pages/DataGenerator/DataGenerator.tsx b/app/client/src/pages/DataGenerator/DataGenerator.tsx index e4c6196d..36f640d1 100644 --- a/app/client/src/pages/DataGenerator/DataGenerator.tsx +++ b/app/client/src/pages/DataGenerator/DataGenerator.tsx @@ -67,35 +67,7 @@ const WizardFooter = styled(Flex)` `; -const steps: WizardStepConfig[] = [ - { - title: 'Configure', - key: DataGenWizardSteps.CONFIGURE, - content: , - required: true, - }, - { - title: 'Examples', - key: DataGenWizardSteps.EXAMPLES, - content: - }, - { - title: 'Prompt', - key: DataGenWizardSteps.PROMPT, - content: , - }, - { - title: 'Summary', - key: DataGenWizardSteps.SUMMARY, - content: - }, - { - title: 'Finish', - key: DataGenWizardSteps.FINISH, - content: - }, - -]; + /** * Wizard component for Synthetic Data Generation workflow @@ -105,10 +77,12 @@ const DataGenerator: FunctionComponent = () => { const [maxStep, setMaxStep] = useState(0); const [isStepValid, setIsStepValid] = useState(false); + // Data passed from listing table to prepopulate form const location = useLocation(); const { generate_file_name } = useParams(); const initialData = location?.state?.data; + const mutation = useMutation({ mutationFn: fetchDatasetDetails }); @@ -118,14 +92,21 @@ const DataGenerator: FunctionComponent = () => { if (generate_file_name && !mutation.data) { mutation.mutate(generate_file_name); } + }, [generate_file_name]); + + useEffect(() => { if (mutation.data && mutation?.data?.dataset) { - form.setFieldsValue({ + const dataset = mutation?.data?.dataset as any; + const values = { ...initialData, - ...(mutation?.data?.dataset as any) - }); + ...dataset, + workflow_type: dataset.technique === 'freeform' ? + WorkflowType.FREE_FORM_DATA_GENERATION : WorkflowType.CUSTOM_DATA_GENERATION + } + form.setFieldsValue(values); + formData.current = values; } - - }, [generate_file_name]); + }, [mutation.data]); if (initialData?.technique) { @@ -157,11 +138,43 @@ const DataGenerator: FunctionComponent = () => { initialData.doc_paths = []; } - const formData = useRef(initialData || { num_questions: 20, topics: [] }); const [form] = Form.useForm(); + + + + const steps: WizardStepConfig[] = [ + { + title: 'Configure', + key: DataGenWizardSteps.CONFIGURE, + content: , + required: true, + }, + { + title: 'Examples', + key: DataGenWizardSteps.EXAMPLES, + content: + }, + { + title: 'Prompt', + key: DataGenWizardSteps.PROMPT, + content: , + }, + { + title: 'Summary', + key: DataGenWizardSteps.SUMMARY, + content: + }, + { + title: 'Finish', + key: DataGenWizardSteps.FINISH, + content: + }, + + ]; + const onStepChange = (value: number) => { setCurrent(value); }; @@ -173,7 +186,7 @@ const DataGenerator: FunctionComponent = () => { } }; - const prev = () => setCurrent(Math.max(0, current - 1)) + const prev = () => setCurrent(Math.max(0, current - 1)); return ( diff --git a/app/client/src/pages/DataGenerator/Examples.tsx b/app/client/src/pages/DataGenerator/Examples.tsx index d0ffe6c3..31e61a60 100644 --- a/app/client/src/pages/DataGenerator/Examples.tsx +++ b/app/client/src/pages/DataGenerator/Examples.tsx @@ -18,6 +18,7 @@ import FreeFormExampleTable from './FreeFormExampleTable'; import { L } from 'vitest/dist/chunks/reporters.DTtkbAtP.js'; import Loading from '../Evaluator/Loading'; import { isEqual, set } from 'lodash'; +import { useParams } from 'react-router-dom'; const { Title, Text } = Typography; const Container = styled.div` @@ -52,6 +53,7 @@ const StyledContainer = styled.div` const Examples: FunctionComponent = () => { const form = Form.useFormInstance(); + const { generate_file_name } = useParams(); const [records, setRecords] = useState[]>([]); const workflowType = form.getFieldValue('workflow_type'); @@ -64,9 +66,13 @@ const Examples: FunctionComponent = () => { }); useEffect(() => { - const useCase = form.getFieldValue('use_case'); - restore_mutation.mutate(useCase); - }, [form.getFieldValue('use_case')]); + if (isEmpty(generate_file_name)) { + const useCase = form.getFieldValue('use_case'); + restore_mutation.mutate(useCase); + } else { + setRecords(form.getFieldValue('examples')); + } + }, [form.getFieldValue('use_case'), generate_file_name]); useEffect(() => { @@ -95,6 +101,12 @@ const Examples: FunctionComponent = () => { setRecords(examples); } }, [restore_mutation.data]); + + useEffect(() => { + if (generate_file_name) { + setRecords(form.getFieldValue('examples')); + } + }, [generate_file_name]); const onRestoreDefaults = async() => { const useCase = form.getFieldValue('use_case'); diff --git a/app/client/src/pages/DataGenerator/Prompt.tsx b/app/client/src/pages/DataGenerator/Prompt.tsx index 45f8b798..f89be152 100644 --- a/app/client/src/pages/DataGenerator/Prompt.tsx +++ b/app/client/src/pages/DataGenerator/Prompt.tsx @@ -226,7 +226,6 @@ const Prompt = () => { } }; } - console.log('mutation data:', mutation); return ( diff --git a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx index 1107bc91..16ae1e08 100644 --- a/app/client/src/pages/DataGenerator/UseCaseSelector.tsx +++ b/app/client/src/pages/DataGenerator/UseCaseSelector.tsx @@ -26,7 +26,6 @@ const UseCaseSelector: FunctionComponent = ({ form }) => { }, [useCasesReq.data]); const onChange = (value: string) => { - console.log('value', value); form.setFieldValue('use_case', value); if (value !== 'custom') { form.setFieldValue('example_path', null); diff --git a/app/client/src/pages/DataGenerator/hooks.ts b/app/client/src/pages/DataGenerator/hooks.ts index c64d0c3f..2980205b 100644 --- a/app/client/src/pages/DataGenerator/hooks.ts +++ b/app/client/src/pages/DataGenerator/hooks.ts @@ -280,7 +280,6 @@ export const fetchExamplesByUseCase = async (use_case: string) => { } export const useGetExamplesByUseCase = (use_case: string) => { - console.log('------------------> useGetExamplesByUseCase', use_case); const { data, isLoading, isError, error, isFetching, refetch } = useQuery( { queryKey: ['fetchUseCaseTopics', fetchExamplesByUseCase], diff --git a/app/client/src/pages/Home/TemplatesSection.tsx b/app/client/src/pages/Home/TemplatesSection.tsx index 69379513..a27b8cd4 100644 --- a/app/client/src/pages/Home/TemplatesSection.tsx +++ b/app/client/src/pages/Home/TemplatesSection.tsx @@ -28,13 +28,11 @@ const StyledContainer = styled.div` const TemplatesSection: React.FC = () => { const useCasesReq = useGetUseCases(); - console.log("useCasesReq", useCasesReq); if (useCasesReq.isLoading) { return ; } const useCases: Template[] = get(useCasesReq, 'data.usecases', []); - console.log("useCases", useCases); return ( From 7feb5745a497b8ced7a5bfc1fd270bf79314afd3 Mon Sep 17 00:00:00 2001 From: Keivan Vosoughi Date: Mon, 28 Jul 2025 20:49:24 -0700 Subject: [PATCH 26/26] Regenerate Issues using PDF Files --- .../src/pages/DataGenerator/Configure.tsx | 26 ++++++++++++++----- .../src/pages/DataGenerator/Examples.tsx | 2 +- app/client/src/pages/DataGenerator/Prompt.tsx | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/app/client/src/pages/DataGenerator/Configure.tsx b/app/client/src/pages/DataGenerator/Configure.tsx index 1326a1eb..85f196bb 100644 --- a/app/client/src/pages/DataGenerator/Configure.tsx +++ b/app/client/src/pages/DataGenerator/Configure.tsx @@ -258,13 +258,27 @@ const Configure: FunctionComponent = () => { } {( - formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || - formData?.workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) && + formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION || + formData?.use_case === 'custom') && + + prevValues.doc_paths !== currentValues.doc_paths || + prevValues.use_case !== currentValues.use_case + } + > + {({}) => { + const useCase = form.getFieldValue('use_case'); + if (useCase === 'custom') { + + } + return ( + { >