diff --git a/src/lib/server/api/routes/groups/user.ts b/src/lib/server/api/routes/groups/user.ts index 21eadca78a3..223c5520da3 100644 --- a/src/lib/server/api/routes/groups/user.ts +++ b/src/lib/server/api/routes/groups/user.ts @@ -71,6 +71,7 @@ export const userGroup = new Elysia() customPrompts: settings?.customPrompts ?? {}, multimodalOverrides: settings?.multimodalOverrides ?? {}, + modelParameters: settings?.modelParameters ?? DEFAULT_SETTINGS.modelParameters, }; }) .post("/settings", async ({ locals, request }) => { diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts index 68d0bec4cd6..25ff7b7dcd3 100644 --- a/src/lib/server/endpoints/openai/endpointOai.ts +++ b/src/lib/server/endpoints/openai/endpointOai.ts @@ -40,7 +40,7 @@ export const endpointOAIParametersSchema = z.object({ "image/jpeg", ], preferredMimeType: "image/jpeg", - maxSizeInMB: 3, + maxSizeInMB: 1, maxWidth: 1024, maxHeight: 1024, }), diff --git a/src/lib/server/textGeneration/generate.ts b/src/lib/server/textGeneration/generate.ts index ae9c608c7c2..3d3737138ad 100644 --- a/src/lib/server/textGeneration/generate.ts +++ b/src/lib/server/textGeneration/generate.ts @@ -13,6 +13,7 @@ export async function* generate( conv, messages, assistant, + userModelParameters, promptedAt, forceMultimodal, locals, @@ -20,10 +21,17 @@ export async function* generate( }: GenerateContext, preprompt?: string ): AsyncIterable { + // Merge parameters with priority: model defaults < user settings < assistant settings + // (model.parameters is merged in the endpoint itself) + const mergedGenerateSettings = { + ...userModelParameters, + ...assistant?.generateSettings, + }; + const stream = await endpoint({ messages, preprompt, - generateSettings: assistant?.generateSettings, + generateSettings: mergedGenerateSettings, // Allow user-level override to force multimodal isMultimodal: (forceMultimodal ?? false) || model.multimodal, conversationId: conv._id, diff --git a/src/lib/server/textGeneration/types.ts b/src/lib/server/textGeneration/types.ts index 791251510c7..237d34bdcf1 100644 --- a/src/lib/server/textGeneration/types.ts +++ b/src/lib/server/textGeneration/types.ts @@ -3,6 +3,7 @@ import type { Endpoint } from "../endpoints/endpoints"; import type { Conversation } from "$lib/types/Conversation"; import type { Message } from "$lib/types/Message"; import type { Assistant } from "$lib/types/Assistant"; +import type { Model } from "$lib/types/Model"; export interface TextGenerationContext { model: ProcessedModel; @@ -10,6 +11,8 @@ export interface TextGenerationContext { conv: Conversation; messages: Message[]; assistant?: Pick; + /** User's per-model parameter overrides from settings */ + userModelParameters?: Partial; promptedAt: Date; ip: string; username?: string; diff --git a/src/lib/stores/settings.ts b/src/lib/stores/settings.ts index 450b2ad4b0d..4335f9d5040 100644 --- a/src/lib/stores/settings.ts +++ b/src/lib/stores/settings.ts @@ -2,6 +2,7 @@ import { browser } from "$app/environment"; import { invalidate } from "$app/navigation"; import { base } from "$app/paths"; import { UrlDependency } from "$lib/types/UrlDependency"; +import type { ModelParameterOverrides } from "$lib/types/Settings"; import { getContext, setContext } from "svelte"; import { type Writable, writable, get } from "svelte/store"; @@ -12,6 +13,7 @@ type SettingsStore = { activeModel: string; customPrompts: Record; multimodalOverrides: Record; + modelParameters: Record; recentlySaved: boolean; disableStream: boolean; directPaste: boolean; @@ -32,7 +34,11 @@ export function useSettingsStore() { } export function createSettingsStore(initialValue: Omit) { - const baseStore = writable({ ...initialValue, recentlySaved: false }); + const baseStore = writable({ + ...initialValue, + modelParameters: initialValue.modelParameters ?? {}, + recentlySaved: false, + }); let timeoutId: NodeJS.Timeout; let showSavedOnNextSync = false; @@ -83,7 +89,7 @@ export function createSettingsStore(initialValue: Omit; + const currentNestedObject = currentStore[key] as Record | undefined; // Only initialize if undefined if (currentNestedObject?.[nestedKey] !== undefined) { @@ -91,14 +97,14 @@ export function createSettingsStore(initialValue: Omit = { ...(currentNestedObject || {}), [nestedKey]: value, }; baseStore.update((s) => ({ ...s, - [key]: newNestedObject, + [key]: newNestedObject as SettingsStore[K], })); // Save to server (debounced) - note: we don't set showSavedOnNextSync diff --git a/src/lib/types/Settings.ts b/src/lib/types/Settings.ts index d988ca8c8d8..06903c8ee4a 100644 --- a/src/lib/types/Settings.ts +++ b/src/lib/types/Settings.ts @@ -2,6 +2,14 @@ import { defaultModel } from "$lib/server/models"; import type { Timestamps } from "./Timestamps"; import type { User } from "./User"; +/** + * Per-model parameter overrides (Tier 1: most commonly customized) + */ +export interface ModelParameterOverrides { + temperature?: number; + max_tokens?: number; +} + export interface Settings extends Timestamps { userId?: User["_id"]; sessionId?: string; @@ -27,6 +35,12 @@ export interface Settings extends Timestamps { */ hidePromptExamples?: Record; + /** + * Per-model parameter customization (temperature, max_tokens, etc.) + * Empty/undefined values fall back to model defaults. + */ + modelParameters?: Record; + disableStream: boolean; directPaste: boolean; } @@ -39,6 +53,7 @@ export const DEFAULT_SETTINGS = { customPrompts: {}, multimodalOverrides: {}, hidePromptExamples: {}, + modelParameters: {}, disableStream: false, directPaste: false, } satisfies SettingsEditable; diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index c74f3c11d37..6d00b03dd67 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -289,7 +289,16 @@ {#if publicConfig.PUBLIC_PLAUSIBLE_SCRIPT_URL} {/if} diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 26467a068f7..73f66fab751 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -355,10 +355,7 @@ export async function POST({ request, locals, params, getClientAddress }) { metrics.model.tokenCountTotal.inc(metricsLabels); if (!firstTokenObserved) { - metrics.model.timeToFirstToken.observe( - metricsLabels, - now - promptedAt.getTime() - ); + metrics.model.timeToFirstToken.observe(metricsLabels, now - promptedAt.getTime()); firstTokenObserved = true; } @@ -468,21 +465,21 @@ export async function POST({ request, locals, params, getClientAddress }) { const initialMessageContent = messageToWriteTo.content; try { + // Fetch user settings once for both multimodal and parameters + const userSettings = await collections.settings.findOne(authCondition(locals)); + const ctx: TextGenerationContext = { model, endpoint: await model.getEndpoint(), conv, messages: messagesForPrompt, assistant: undefined, + userModelParameters: userSettings?.modelParameters?.[model.id], promptedAt, ip: getClientAddress(), username: locals.user?.username, // Force-enable multimodal if user settings say so for this model - forceMultimodal: Boolean( - (await collections.settings.findOne(authCondition(locals)))?.multimodalOverrides?.[ - model.id - ] - ), + forceMultimodal: Boolean(userSettings?.multimodalOverrides?.[model.id]), locals, abortController: ctrl, }; diff --git a/src/routes/settings/(nav)/+layout.svelte b/src/routes/settings/(nav)/+layout.svelte index 4e5994b8b89..1750afb364c 100644 --- a/src/routes/settings/(nav)/+layout.svelte +++ b/src/routes/settings/(nav)/+layout.svelte @@ -6,9 +6,10 @@ import { useSettingsStore } from "$lib/stores/settings"; import IconOmni from "$lib/components/icons/IconOmni.svelte"; import CarbonClose from "~icons/carbon/close"; - import CarbonTextLongParagraph from "~icons/carbon/text-long-paragraph"; + import CarbonChat from "~icons/carbon/chat"; import CarbonChevronLeft from "~icons/carbon/chevron-left"; import CarbonView from "~icons/carbon/view"; + import CarbonToolsAlt from "~icons/carbon/tools-alt"; import IconGear from "~icons/bi/gear-fill"; import type { LayoutData } from "../$types"; @@ -185,8 +186,15 @@ {/if} {#if $settings.customPrompts?.[model.id]} - + {/if} + {#if $settings.modelParameters?.[model.id] && Object.keys($settings.modelParameters[model.id]).length > 0} + {/if} {#if model.id === $settings.activeModel} diff --git a/src/routes/settings/(nav)/+server.ts b/src/routes/settings/(nav)/+server.ts index 3222cb58d3a..285b80f026e 100644 --- a/src/routes/settings/(nav)/+server.ts +++ b/src/routes/settings/(nav)/+server.ts @@ -15,6 +15,14 @@ export async function POST({ request, locals }) { activeModel: z.string().default(DEFAULT_SETTINGS.activeModel), customPrompts: z.record(z.string()).default({}), multimodalOverrides: z.record(z.boolean()).default({}), + modelParameters: z + .record( + z.object({ + temperature: z.number().min(0).max(2).optional(), + max_tokens: z.number().int().positive().optional(), + }) + ) + .default({}), disableStream: z.boolean().default(false), directPaste: z.boolean().default(false), hidePromptExamples: z.record(z.boolean()).default({}), diff --git a/src/routes/settings/(nav)/[...model]/+page.svelte b/src/routes/settings/(nav)/[...model]/+page.svelte index f4bbd8b4350..4d828ec19c9 100644 --- a/src/routes/settings/(nav)/[...model]/+page.svelte +++ b/src/routes/settings/(nav)/[...model]/+page.svelte @@ -3,12 +3,13 @@ import { base } from "$app/paths"; import type { BackendModel } from "$lib/server/models"; + import type { ModelParameterOverrides } from "$lib/types/Settings"; import IconOmni from "$lib/components/icons/IconOmni.svelte"; import { useSettingsStore } from "$lib/stores/settings"; import CopyToClipBoardBtn from "$lib/components/CopyToClipBoardBtn.svelte"; import CarbonArrowUpRight from "~icons/carbon/arrow-up-right"; import CarbonCopy from "~icons/carbon/copy"; - import CarbonChat from "~icons/carbon/chat"; + import IconNew from "$lib/components/icons/IconNew.svelte"; import CarbonCode from "~icons/carbon/code"; import { goto } from "$app/navigation"; @@ -27,8 +28,10 @@ }); let hasCustomPreprompt = $derived( - $settings.customPrompts[page.params.model] !== - page.data.models.find((el: BackendModel) => el.id === page.params.model)?.preprompt + $settings.customPrompts[page.params.model] !== undefined && + $settings.customPrompts[page.params.model] !== "" && + $settings.customPrompts[page.params.model] !== + page.data.models.find((el: BackendModel) => el.id === page.params.model)?.preprompt ); let model = $derived(page.data.models.find((el: BackendModel) => el.id === page.params.model)); @@ -46,6 +49,104 @@ $effect(() => { settings.initValue("hidePromptExamples", page.params.model, false); }); + + let modelId = $derived(page.params.model); + + // Track whether model parameters details is expanded per model + let parametersExpanded = $state(false); + + type ParameterKey = keyof ModelParameterOverrides; + + type ParameterConfig = { + key: ParameterKey; + label: string; + description: string; + min: number; + max?: number; + step: number; + integer?: boolean; + }; + + const parameterConfigs: ParameterConfig[] = [ + { + key: "temperature", + label: "Temperature", + description: "Controls randomness (low = focused, high = creative)", + min: 0, + max: 2, + step: 0.1, + }, + { + key: "max_tokens", + label: "Max Tokens", + description: "Maximum number of total generated tokens", + min: 1, + step: 1, + integer: true, + }, + ]; + + function hasCustomParameter(param: ParameterKey) { + const overrides = $settings.modelParameters?.[modelId]; + if (!overrides) return false; + + const userValue = overrides[param]; + const modelDefault = model?.parameters?.[param]; + return userValue !== undefined && userValue !== modelDefault; + } + + function updateParameter(param: ParameterKey, value: number | undefined) { + const normalized = value === undefined || Number.isNaN(value) ? undefined : value; + const currentOverrides = $settings.modelParameters?.[modelId]; + + if (normalized === undefined) { + if (!currentOverrides || currentOverrides[param] === undefined) { + return; + } + } else if (currentOverrides?.[param] === normalized) { + return; + } + + settings.update((state) => { + const overridesInState = state.modelParameters?.[modelId]; + const updatedOverrides: ModelParameterOverrides = { ...(overridesInState ?? {}) }; + + if (normalized === undefined) { + delete updatedOverrides[param]; + } else { + updatedOverrides[param] = normalized; + } + + const nextModelParameters = { ...(state.modelParameters ?? {}) }; + + if (Object.keys(updatedOverrides).length === 0) { + delete nextModelParameters[modelId]; + } else { + nextModelParameters[modelId] = updatedOverrides; + } + + return { + ...state, + modelParameters: nextModelParameters, + }; + }); + } + + function handleParameterInput(param: ParameterKey, rawValue: string, integer = false) { + const trimmed = rawValue.trim(); + + if (trimmed === "") { + updateParameter(param, undefined); + return; + } + + const parsed = integer ? Number.parseInt(trimmed, 10) : Number.parseFloat(trimmed); + if (Number.isNaN(parsed)) { + return; + } + + updateParameter(param, parsed); + }
@@ -74,7 +175,7 @@ goto(`${base}/`); }} > - + New chat @@ -174,11 +275,68 @@ class="w-full resize-none rounded-md border border-gray-200 bg-gray-50 p-2 text-[13px] dark:border-gray-700 dark:bg-gray-900 dark:text-gray-200" bind:value={$settings.customPrompts[page.params.model]} > +
+ {#if !model?.isRouter} +
+ +
+
+ Model Parameters +
+

+ Customize generation parameters. +

+
+ + +
+ {#each parameterConfigs as config (config.key)} +
+
+ + {#if hasCustomParameter(config.key)} + + {/if} +
+ + handleParameterInput( + config.key, + e.currentTarget.value, + Boolean(config.integer) + )} + class="w-full rounded border border-gray-200 px-2 py-1 text-[13px] dark:border-gray-700 dark:bg-gray-900" + /> +

+ {config.description} +

+
+ {/each} +
+
+ {/if} +