Merge pull request #113 from cloudera/DSE-47416

DSE-47416 - Add Model Provider UI

Author: vkeivan

app/client/src/Container.tsx

@@ -95,6 +95,12 @@ const pages: MenuItem[] = [
       <Link to={`${Pages.EXPORTS}`}>{LABELS[Pages.EXPORTS]}</Link>
     ),
   },
+  {
+    key: Pages.SETTINGS,
+    label: (
+      <Link to={`${Pages.SETTINGS}`}>{LABELS[Pages.SETTINGS]}</Link>
+    ),
+  },
 
   // {
   //   key: Pages.TELEMETRY,  

app/client/src/components/JobStatus/jobStatusIcon.tsx

@@ -24,6 +24,12 @@ const IconWrapper = styled.div`
     }
 `
 
+const StyledIconWrapper = styled(IconWrapper)`
+    svg {
+        color: #008cff;
+    }
+`;
+
 
 export default function JobStatusIcon({ status, customTooltipTitles }: JobStatusProps) {
     const tooltipTitles = {...defaultTooltipTitles, ...customTooltipTitles};
@@ -44,11 +50,11 @@ export default function JobStatusIcon({ status, customTooltipTitles }: JobStatus
                     </Tooltip>;
             case 'ENGINE_SCHEDULING':
                 return <Tooltip title={tooltipTitles.ENGINE_SCHEDULING}>
-                    <IconWrapper><LoadingOutlined spin/></IconWrapper>
+                    <StyledIconWrapper><LoadingOutlined spin/></StyledIconWrapper>
                     </Tooltip>;
             case 'ENGINE_RUNNING':
                 return <Tooltip title={tooltipTitles.ENGINE_RUNNING}>
-                    <IconWrapper><LoadingOutlined spin /></IconWrapper>
+                    <StyledIconWrapper><LoadingOutlined spin /></StyledIconWrapper>
                     </Tooltip>;
             case null:
                 return <Tooltip title={tooltipTitles.null}>

app/client/src/constants.ts

@@ -12,6 +12,7 @@ export const LABELS = {
     [Pages.EXPORTS]: 'Exports',
     [Pages.HISTORY]: 'History',
     [Pages.FEEDBACK]: 'Feedback',
+    [Pages.SETTINGS]: 'Settings',
     //[Pages.TELEMETRY]: 'Telemetry',
     [ModelParameters.TEMPERATURE]: 'Temperature',
     [ModelParameters.TOP_K]: 'Top K',

app/client/src/pages/DataGenerator/Configure.tsx

@@ -2,17 +2,23 @@ import endsWith from 'lodash/endsWith';
 import isEmpty from 'lodash/isEmpty';
 import isFunction from 'lodash/isFunction';
 import { FunctionComponent, useEffect, useState } from 'react';
-import { Flex, Form, FormInstance, Input, Select, Typography } from 'antd';
+import { Flex, Form, Input, Select, Typography } from 'antd';
 import styled from 'styled-components';
 import { File, WorkflowType } from './types';
 import { useFetchModels } from '../../api/api';
 import { MODEL_PROVIDER_LABELS } from './constants';
 import { ModelProviders, ModelProvidersDropdownOpts } from './types';
-import { getWizardModel, getWizardModeType, useWizardCtx } from './utils';
+import { getWizardModeType, useWizardCtx } from './utils';
 import FileSelectorButton from './FileSelectorButton';
 import UseCaseSelector from './UseCaseSelector';
 import { useLocation, useParams } from 'react-router-dom';
 import { WizardModeType } from '../../types';
+import get from 'lodash/get';
+import forEach from 'lodash/forEach';
+import { useModelProviders } from '../Settings/hooks';
+import { ModelProviderType } from '../Settings/AddModelProviderButton';
+import { CustomModel } from '../Settings/SettingsPage';
+import filter from 'lodash/filter';
 
 
 const StepContainer = styled(Flex)`
@@ -47,14 +53,21 @@ export const WORKFLOW_OPTIONS = [
 export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [
     { label: MODEL_PROVIDER_LABELS[ModelProviders.BEDROCK], value: ModelProviders.BEDROCK},
     { label: MODEL_PROVIDER_LABELS[ModelProviders.CAII], value: ModelProviders.CAII },
+    { label: MODEL_PROVIDER_LABELS[ModelProviders.OPENAI], value: ModelProviders.OPENAI },
+    { label: MODEL_PROVIDER_LABELS[ModelProviders.GEMINI], value: ModelProviders.GEMINI },
 ];
 
 const Configure: FunctionComponent = () => {
     const form = Form.useFormInstance();
     const formData = Form.useWatch((values) => values, form);
     const location = useLocation();
     const { template_name, generate_file_name } = useParams();
+    const [models, setModels] = useState<string[]>([])
     const [wizardModeType, setWizardModeType] = useState(getWizardModeType(location));
+    const { data } = useFetchModels();
+    const customModelPrividersReq = useModelProviders();  
+    const customModels = get(customModelPrividersReq, 'data.endpoints', []);
+    console.log('customModels', customModels);
 
     useEffect(() => {
         if (wizardModeType === WizardModeType.DATA_AUGMENTATION) {
@@ -77,10 +90,18 @@ const Configure: FunctionComponent = () => {
         }
     }, [template_name]);
 
+    useEffect(() => {
+        // set model providers
+        // set model ids
+        if (formData && (formData?.inference_type === ModelProviderType.OPENAI || formData?.inference_type === ModelProviderType.GEMINI) && isEmpty(generate_file_name)) {
+            form.setFieldValue('inference_type', ModelProviders.OPENAI);
+        }
+
+    }, [customModels, formData]);
+
 
     // let formData = Form.useWatch((values) => values, form);
     const { setIsStepValid } = useWizardCtx();
-    const { data } = useFetchModels();
     const [selectedFiles, setSelectedFiles] = useState(
         !isEmpty(form.getFieldValue('doc_paths')) ? form.getFieldValue('doc_paths') : []);
 
@@ -104,7 +125,6 @@ const Configure: FunctionComponent = () => {
 
 
     useEffect(() => {
-        console.log('useEffect 1');
         if (formData && formData?.inference_type === undefined && isEmpty(generate_file_name)) {
             form.setFieldValue('inference_type', ModelProviders.CAII);
             setTimeout(() => {
@@ -155,6 +175,20 @@ const Configure: FunctionComponent = () => {
         }
     }
 
+    const onModelProviderChange = (value: string) => {
+        form.setFieldValue('model_id', undefined)
+        console.log('value', value);
+        if (ModelProviderType.OPENAI === value) {
+            const _models = filter(customModels, (model: CustomModel) => model.provider_type === ModelProviderType.OPENAI);
+            setModels(_models.map((_model: CustomModel) => _model.model_id));  
+        } else if (ModelProviderType.GEMINI === value) {
+            const _models = filter(customModels, (model: CustomModel) => model.provider_type === ModelProviderType.GEMINI);
+            setModels(_models.map((_model: CustomModel) => _model.model_id));   
+        }
+    }
+    console.log('models', models);
+    
+
     return (
         <StepContainer justify='center'>
             <FormContainer vertical>
@@ -178,7 +212,7 @@ const Configure: FunctionComponent = () => {
                 >
                     <Select
 
-                        onChange={() => form.setFieldValue('model_id', undefined)}
+                        onChange={(value: string) => onModelProviderChange(value)}
                         placeholder={'Select a model provider'}
                     >
                         {MODEL_TYPE_OPTIONS.map(({ label, value }, i) =>
@@ -200,15 +234,22 @@ const Configure: FunctionComponent = () => {
                     {formData?.inference_type === ModelProviders.CAII ? (
                         <Input placeholder={'Enter Cloudera AI Inference Model ID'}/>
                     ) : (
-                        <Select placeholder={'Select a Model'} notFoundContent={'You must select a Model Provider before selecting a Model'}>
-                            {!isEmpty(data?.models) && data?.models[ModelProviders.BEDROCK]?.map((model, i) =>
+                        <Select
+                            placeholder={'Select a Model'}
+                            notFoundContent={'You must select a Model Provider before selecting a Model'}
+                        >
+                            {formData?.inference_type === ModelProviders.BEDROCK  && data?.models?.[ModelProviders.BEDROCK]?.map((model, i) => (
                                 <Select.Option key={`${model}-${i}`} value={model}>
                                     {model}
                                 </Select.Option>
-                            )}
+                            ))}
+                            {(formData?.inference_type === ModelProviders.OPENAI || formData?.inference_type === ModelProviders.GEMINI) && models?.map((model, i) => (
+                                <Select.Option key={`${model}-${i}`} value={model}>
+                                    {model}
+                                </Select.Option>
+                            ))}
                         </Select>
                     )}
-
                 </Form.Item>
                 {formData?.inference_type === ModelProviders.CAII && (
                     <>

app/client/src/pages/DataGenerator/Examples.tsx

@@ -132,12 +132,13 @@ const Examples: FunctionComponent = () => {
     };
 
     const showEmptyState = (workflowType === WorkflowType.FREE_FORM_DATA_GENERATION && 
-        isEmpty(mutation.data) &&
+        isEmpty(mutation.data) && Array.isArray(records) &&
         records.length === 0) || 
         (form.getFieldValue('use_case') === 'custom' && 
         isEmpty(form.getFieldValue('examples')));
 
 
+    console.log('records', records);    
     return (
         <Container>
             {mutation?.isPending || restore_mutation.isPending && <Loading />}

app/client/src/pages/DataGenerator/Finish.tsx

@@ -322,12 +322,14 @@ const Finish = () => {
     const rawData = genDatasetResp !== null && hasTopics(genDatasetResp) ?  
         getRawData(genDatasetResp) : genDatasetResp?.results
 
+    console.log('Finish >> ', isDemo);    
+
     return (
         <div>
             <Title level={2}>
                 <Flex align='center' gap={10}>
                     <CheckCircleIcon style={{ color: '#178718' }}/>
-                    {'Success'}
+                    {isDemo ? 'Success' : 'Job Successfully Started'}
                 </Flex>
             </Title>
             {isDemo ? (

app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx

@@ -48,6 +48,7 @@ interface Props {
 const FreeFormExampleTable: FunctionComponent<Props> = ({ data }) => {
     const [colDefs, setColDefs] = useState([]);
     const [rowData, setRowData] = useState([]);
+    console.log('FreeFormExampleTable', data);
 
     useEffect(() => {
         if (!isEmpty(data)) {

app/client/src/pages/DataGenerator/Success.tsx

@@ -122,7 +122,7 @@ const Success: FC<SuccessProps> = ({ formData, isDemo = true }) => {
             <Title level={2}>
                 <Flex align='center' gap={10}>
                     <CheckCircleIcon style={{ color: '#178718' }}/>
-                    {'Success'}
+                    {isDemo ? 'Success' : 'Job successfully started.'}
                 </Flex>
             </Title>
             {isDemo ? (

app/client/src/pages/DataGenerator/constants.ts

@@ -5,6 +5,8 @@ export const MODEL_PROVIDER_LABELS = {
   [ModelProviders.CAII]: 'Cloudera AI Inference Service',
   [ModelProviders.GOOGLE_GEMINI]: 'Google Gemini',
   [ModelProviders.AZURE_OPENAI]: 'Azure OpenAI',
+  [ModelProviders.GEMINI]: 'Gemini',
+  [ModelProviders.OPENAI]: 'OpenAI'
 };
 
 export const MIN_SEED_INSTRUCTIONS = 1

app/client/src/pages/DataGenerator/types.ts

@@ -19,6 +19,8 @@ export enum ModelProviders {
     CAII = 'CAII',
     AZURE_OPENAI = 'AZURE_OPENAI',
     GOOGLE_GEMINI = 'GOOGLE_GEMINI',
+    OPENAI = 'openai',
+    GEMINI = 'gemini',
 }
 
 export type ModelProvidersDropdownOpts = { label: string, value: ModelProviders }[];