Add bedrock model providers

.gitignore

@@ -55,6 +55,16 @@ qa_pairs*
 Khauneesh/
 *job_args*
 
+# Generated data files
+freeform_data_*.json
+row_data_*.json
+lending_*.json
+seeds_*.json
+SeedsInstructions.json
+*_example.json
+nm.json
+french_input.json
+
 # DB
 *metadata.db-shm
 *metadata.db-wal

.project-metadata.yaml

@@ -42,6 +42,10 @@ environment_variables:
     default: null
     description: >-
       Gemini API Key. Check the Google Gemini documentation for information about role access
+  OpenAI_Endpoint_Compatible_Key:
+    default: null
+    description: >-
+      API Key for OpenAI Compatible endpoints. Used for custom OpenAI-compatible model endpoints.
 # runtimes
 runtimes:
   - editor: JupyterLab

app/client/src/Container.tsx

@@ -95,6 +95,12 @@ const pages: MenuItem[] = [
       <Link to={`${Pages.EXPORTS}`}>{LABELS[Pages.EXPORTS]}</Link>
     ),
   },
+  {
+    key: Pages.SETTINGS,
+    label: (
+      <Link to={`${Pages.SETTINGS}`}>{LABELS[Pages.SETTINGS]}</Link>
+    ),
+  },
 
   // {
   //   key: Pages.TELEMETRY,  

app/client/src/components/JobStatus/jobStatusIcon.tsx

@@ -24,6 +24,12 @@ const IconWrapper = styled.div`
     }
 `
 
+const StyledIconWrapper = styled(IconWrapper)`
+    svg {
+        color: #008cff;
+    }
+`;
+
 
 export default function JobStatusIcon({ status, customTooltipTitles }: JobStatusProps) {
     const tooltipTitles = {...defaultTooltipTitles, ...customTooltipTitles};
@@ -44,11 +50,11 @@ export default function JobStatusIcon({ status, customTooltipTitles }: JobStatus
                     </Tooltip>;
             case 'ENGINE_SCHEDULING':
                 return <Tooltip title={tooltipTitles.ENGINE_SCHEDULING}>
-                    <IconWrapper><LoadingOutlined spin/></IconWrapper>
+                    <StyledIconWrapper><LoadingOutlined spin/></StyledIconWrapper>
                     </Tooltip>;
             case 'ENGINE_RUNNING':
                 return <Tooltip title={tooltipTitles.ENGINE_RUNNING}>
-                    <IconWrapper><LoadingOutlined spin /></IconWrapper>
+                    <StyledIconWrapper><LoadingOutlined spin /></StyledIconWrapper>
                     </Tooltip>;
             case null:
                 return <Tooltip title={tooltipTitles.null}>

app/client/src/constants.ts

@@ -12,6 +12,7 @@ export const LABELS = {
     [Pages.EXPORTS]: 'Exports',
     [Pages.HISTORY]: 'History',
     [Pages.FEEDBACK]: 'Feedback',
+    [Pages.SETTINGS]: 'Settings',
     //[Pages.TELEMETRY]: 'Telemetry',
     [ModelParameters.TEMPERATURE]: 'Temperature',
     [ModelParameters.TOP_K]: 'Top K',

app/client/src/pages/DataGenerator/Configure.tsx

@@ -2,17 +2,23 @@ import endsWith from 'lodash/endsWith';
 import isEmpty from 'lodash/isEmpty';
 import isFunction from 'lodash/isFunction';
 import { FunctionComponent, useEffect, useState } from 'react';
-import { Flex, Form, FormInstance, Input, Select, Typography } from 'antd';
+import { Flex, Form, Input, Select, Typography } from 'antd';
 import styled from 'styled-components';
 import { File, WorkflowType } from './types';
 import { useFetchModels } from '../../api/api';
 import { MODEL_PROVIDER_LABELS } from './constants';
 import { ModelProviders, ModelProvidersDropdownOpts } from './types';
-import { getWizardModel, getWizardModeType, useWizardCtx } from './utils';
+import { getWizardModeType, useWizardCtx } from './utils';
 import FileSelectorButton from './FileSelectorButton';
 import UseCaseSelector from './UseCaseSelector';
 import { useLocation, useParams } from 'react-router-dom';
 import { WizardModeType } from '../../types';
+import get from 'lodash/get';
+import forEach from 'lodash/forEach';
+import { useModelProviders } from '../Settings/hooks';
+import { ModelProviderType } from '../Settings/AddModelProviderButton';
+import { CustomModel } from '../Settings/SettingsPage';
+import filter from 'lodash/filter';
 
 
 const StepContainer = styled(Flex)`
@@ -47,14 +53,21 @@ export const WORKFLOW_OPTIONS = [
 export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [
     { label: MODEL_PROVIDER_LABELS[ModelProviders.BEDROCK], value: ModelProviders.BEDROCK},
     { label: MODEL_PROVIDER_LABELS[ModelProviders.CAII], value: ModelProviders.CAII },
+    { label: MODEL_PROVIDER_LABELS[ModelProviders.OPENAI], value: ModelProviders.OPENAI },
+    { label: MODEL_PROVIDER_LABELS[ModelProviders.GEMINI], value: ModelProviders.GEMINI },
 ];
 
 const Configure: FunctionComponent = () => {
     const form = Form.useFormInstance();
     const formData = Form.useWatch((values) => values, form);
     const location = useLocation();
     const { template_name, generate_file_name } = useParams();
+    const [models, setModels] = useState<string[]>([])
     const [wizardModeType, setWizardModeType] = useState(getWizardModeType(location));
+    const { data } = useFetchModels();
+    const customModelPrividersReq = useModelProviders();  
+    const customModels = get(customModelPrividersReq, 'data.endpoints', []);
+    console.log('customModels', customModels);
 
     useEffect(() => {
         if (wizardModeType === WizardModeType.DATA_AUGMENTATION) {
@@ -77,10 +90,18 @@ const Configure: FunctionComponent = () => {
         }
     }, [template_name]);
 
+    useEffect(() => {
+        // set model providers
+        // set model ids
+        if (formData && (formData?.inference_type === ModelProviderType.OPENAI || formData?.inference_type === ModelProviderType.GEMINI) && isEmpty(generate_file_name)) {
+            form.setFieldValue('inference_type', ModelProviders.OPENAI);
+        }
+
+    }, [customModels, formData]);
+
 
     // let formData = Form.useWatch((values) => values, form);
     const { setIsStepValid } = useWizardCtx();
-    const { data } = useFetchModels();
     const [selectedFiles, setSelectedFiles] = useState(
         !isEmpty(form.getFieldValue('doc_paths')) ? form.getFieldValue('doc_paths') : []);
 
@@ -104,7 +125,6 @@ const Configure: FunctionComponent = () => {
 
 
     useEffect(() => {
-        console.log('useEffect 1');
         if (formData && formData?.inference_type === undefined && isEmpty(generate_file_name)) {
             form.setFieldValue('inference_type', ModelProviders.CAII);
             setTimeout(() => {
@@ -155,6 +175,20 @@ const Configure: FunctionComponent = () => {
         }
     }
 
+    const onModelProviderChange = (value: string) => {
+        form.setFieldValue('model_id', undefined)
+        console.log('value', value);
+        if (ModelProviderType.OPENAI === value) {
+            const _models = filter(customModels, (model: CustomModel) => model.provider_type === ModelProviderType.OPENAI);
+            setModels(_models.map((_model: CustomModel) => _model.model_id));  
+        } else if (ModelProviderType.GEMINI === value) {
+            const _models = filter(customModels, (model: CustomModel) => model.provider_type === ModelProviderType.GEMINI);
+            setModels(_models.map((_model: CustomModel) => _model.model_id));   
+        }
+    }
+    console.log('models', models);
+
+
     return (
         <StepContainer justify='center'>
             <FormContainer vertical>
@@ -178,7 +212,7 @@ const Configure: FunctionComponent = () => {
                 >
                     <Select
 
-                        onChange={() => form.setFieldValue('model_id', undefined)}
+                        onChange={(value: string) => onModelProviderChange(value)}
                         placeholder={'Select a model provider'}
                     >
                         {MODEL_TYPE_OPTIONS.map(({ label, value }, i) =>
@@ -200,15 +234,22 @@ const Configure: FunctionComponent = () => {
                     {formData?.inference_type === ModelProviders.CAII ? (
                         <Input placeholder={'Enter Cloudera AI Inference Model ID'}/>
                     ) : (
-                        <Select placeholder={'Select a Model'} notFoundContent={'You must select a Model Provider before selecting a Model'}>
-                            {!isEmpty(data?.models) && data?.models[ModelProviders.BEDROCK]?.map((model, i) =>
+                        <Select
+                            placeholder={'Select a Model'}
+                            notFoundContent={'You must select a Model Provider before selecting a Model'}
+                        >
+                            {formData?.inference_type === ModelProviders.BEDROCK  && data?.models?.[ModelProviders.BEDROCK]?.map((model, i) => (
                                 <Select.Option key={`${model}-${i}`} value={model}>
                                     {model}
                                 </Select.Option>
-                            )}
+                            ))}
+                            {(formData?.inference_type === ModelProviders.OPENAI || formData?.inference_type === ModelProviders.GEMINI) && models?.map((model, i) => (
+                                <Select.Option key={`${model}-${i}`} value={model}>
+                                    {model}
+                                </Select.Option>
+                            ))}
                         </Select>
                     )}
-
                 </Form.Item>
                 {formData?.inference_type === ModelProviders.CAII && (
                     <>

app/client/src/pages/DataGenerator/Examples.tsx

@@ -132,12 +132,13 @@ const Examples: FunctionComponent = () => {
     };
 
     const showEmptyState = (workflowType === WorkflowType.FREE_FORM_DATA_GENERATION && 
-        isEmpty(mutation.data) &&
+        isEmpty(mutation.data) && Array.isArray(records) &&
         records.length === 0) || 
         (form.getFieldValue('use_case') === 'custom' && 
         isEmpty(form.getFieldValue('examples')));
 
 
+    console.log('records', records);    
     return (
         <Container>
             {mutation?.isPending || restore_mutation.isPending && <Loading />}

app/client/src/pages/DataGenerator/Finish.tsx

@@ -255,7 +255,8 @@ const Finish = () => {
             title: 'Review Dataset',
             description: 'Review your dataset to ensure it properly fits your usecase.',
             icon: <GradingIcon/>,
-            href: getFilesURL(genDatasetResp?.export_path?.local || "")
+            href: getFilesURL(genDatasetResp?.export_path?.local || ""),
+            external: true
         },
         {
             avatar: '',
@@ -278,7 +279,8 @@ const Finish = () => {
             title: 'Review Dataset',
             description: 'Once your dataset finishes generating, you can review your dataset in the workbench files',
             icon: <GradingIcon/>,
-            href: getFilesURL('')
+            href: getFilesURL(''),
+            external: true
         },
         {
             avatar: '',
@@ -320,12 +322,14 @@ const Finish = () => {
     const rawData = genDatasetResp !== null && hasTopics(genDatasetResp) ?  
         getRawData(genDatasetResp) : genDatasetResp?.results
 
+    console.log('Finish >> ', isDemo);    
+
     return (
         <div>
             <Title level={2}>
                 <Flex align='center' gap={10}>
                     <CheckCircleIcon style={{ color: '#178718' }}/>
-                    {'Success'}
+                    {isDemo ? 'Success' : 'Job Successfully Started'}
                 </Flex>
             </Title>
             {isDemo ? (
@@ -361,7 +365,18 @@ const Finish = () => {
             <List
                 itemLayout="horizontal"
                 dataSource={isDemo ? nextStepsListPreview : nextStepsListNonPreview}
-                renderItem={({ title, href, icon, description}, i) => (
+                renderItem={({ title, href, icon, description, external }, i) => (
+                    external ? 
+                    <Link to={href} target="_blank" rel="noopener noreferrer">
+                        <List.Item key={`${title}-${i}`}>
+                            <List.Item.Meta
+                                avatar={<Avatar style={{ backgroundColor: '#1677ff'}} icon={icon} />}
+                                title={title}
+                                description={description}
+                            />
+                        </List.Item>
+                    </Link> : 
+
                     <Link to={href}>
                         <List.Item key={`${title}-${i}`}>
                             <List.Item.Meta

app/client/src/pages/DataGenerator/FreeFormExampleTable.tsx

@@ -48,6 +48,7 @@ interface Props {
 const FreeFormExampleTable: FunctionComponent<Props> = ({ data }) => {
     const [colDefs, setColDefs] = useState([]);
     const [rowData, setRowData] = useState([]);
+    console.log('FreeFormExampleTable', data);
 
     useEffect(() => {
         if (!isEmpty(data)) {

app/client/src/pages/DataGenerator/Success.tsx

@@ -122,7 +122,7 @@ const Success: FC<SuccessProps> = ({ formData, isDemo = true }) => {
             <Title level={2}>
                 <Flex align='center' gap={10}>
                     <CheckCircleIcon style={{ color: '#178718' }}/>
-                    {'Success'}
+                    {isDemo ? 'Success' : 'Job successfully started.'}
                 </Flex>
             </Title>
             {isDemo ? (

app/client/src/pages/DataGenerator/constants.ts

@@ -5,6 +5,8 @@ export const MODEL_PROVIDER_LABELS = {
   [ModelProviders.CAII]: 'Cloudera AI Inference Service',
   [ModelProviders.GOOGLE_GEMINI]: 'Google Gemini',
   [ModelProviders.AZURE_OPENAI]: 'Azure OpenAI',
+  [ModelProviders.GEMINI]: 'Gemini',
+  [ModelProviders.OPENAI]: 'OpenAI'
 };
 
 export const MIN_SEED_INSTRUCTIONS = 1

app/client/src/pages/DataGenerator/types.ts

@@ -19,6 +19,8 @@ export enum ModelProviders {
     CAII = 'CAII',
     AZURE_OPENAI = 'AZURE_OPENAI',
     GOOGLE_GEMINI = 'GOOGLE_GEMINI',
+    OPENAI = 'openai',
+    GEMINI = 'gemini',
 }
 
 export type ModelProvidersDropdownOpts = { label: string, value: ModelProviders }[];

app/client/src/pages/Datasets/DatasetsPage.tsx

@@ -4,7 +4,6 @@ import { Col, Flex, Input, Layout, Row, Table, TableProps, Tooltip, notification
 import styled from 'styled-components';
 import Paragraph from 'antd/es/typography/Paragraph';
 import { useDatasets } from '../Home/hooks';
-import { ExportResult } from '../../components/Export/ExportModal';
 import { SearchProps } from 'antd/es/input';
 import Loading from '../Evaluator/Loading';
 import { Dataset } from '../Evaluator/types';

app/client/src/pages/Home/DatasetsTab.tsx

@@ -106,10 +106,16 @@ const DatasetsTab: React.FC<Props> = ({ hideSearch = false }) => {
             key: 'job_status',
             title: 'Status',
             dataIndex: 'job_status',
-            width: 80,
+            width: 140,
             sorter: sortItemsByKey('job_status'),
-            render: (status: JobStatus) => <Flex justify='center' align='center'>
+            render: (status: JobStatus) => 
+            <Flex>
                 <JobStatusIcon status={status} customTooltipTitles={{"null": `Job wasn't executed because dataset total count is less than ${JOB_EXECUTION_TOTAL_COUNT_THRESHOLD}!`}}></JobStatusIcon>
+                {status === 'ENGINE_SCHEDULING' && <div style={{ marginLeft: '5px'}}>{'Scheduling'}</div> }
+                {status === 'ENGINE_RUNNING' && <div style={{ marginLeft: '5px'}}>{'Running'}</div> }
+                {status === 'ENGINE_STOPPED' && <div style={{ marginLeft: '5px'}}>{'Stopped'}</div> }
+                {status === 'ENGINE_SUCCEEDED' && <div style={{ marginLeft: '5px'}}>{'Success'}</div> }
+                {status === 'ENGINE_TIMEDOUT' && <div style={{ marginLeft: '5px'}}>{'Timeout'}</div> }
             </Flex>
         },
         {

app/client/src/pages/Home/TemplateCard.tsx

@@ -132,10 +132,10 @@ const TagsContainer = styled.div`
   }
 `;
 
-const StyledTag = styled(Tag)`
-  color: ${props => props.theme.color};
-  background-color: ${props => props.theme.backgroundColor};
-  border: 1px solid ${props => props.theme.borderColor};
+const StyledTag = styled(Tag)<{ $theme: { color: string; backgroundColor: string; borderColor: string } }>`
+  color: ${props => props.$theme.color} !important;
+  background-color: ${props => props.$theme.backgroundColor} !important;
+  border: 1px solid ${props => props.$theme.borderColor} !important;
 `;
 
 
@@ -150,7 +150,7 @@ const TemplateCard: React.FC<Props> = ({ template }) => {
         const { color, backgroundColor, borderColor } = getTemplateTagColors(theme as string);
 
         return (
-          <StyledTag key={tag} theme={{ color, backgroundColor, borderColor }}>
+          <StyledTag key={tag} $theme={{ color, backgroundColor, borderColor }}>
             <div className="tag-title" title={tag} style={{ maxWidth: '150px', color }}>
                 {tag}
             </div>