khalidadil/dynamic-sites-simple
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to a smaller intent model

Browse Source
This commit is contained in:
khalid@traclabs.com
2026-04-23 08:26:24 -05:00
parent 36bce5a908
commit 4e014fa648
2 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/llm/client.ts
View File

@@ -6,6 +6,7 @@ const OLLAMA_HOST = process.env.OLLAMA_HOST || 'http://localhost:11434';
const OLLAMA_API_KEY = process.env.OLLAMA_API_KEY || '';
const PRIMARY_MODEL = process.env.OLLAMA_MODEL || 'qwen3.5:397b-cloud';
const FALLBACK_MODEL = process.env.OLLAMA_FALLBACK_MODEL || 'gpt-oss:120b';
const INTENT_MODEL = process.env.OLLAMA_INTENT_MODEL || 'gemma4:31b-cloud';
const MAX_RETRIES = 3;
export interface LlmChatCaller {
@@ -39,9 +40,10 @@ async function generateWithValidation<T>(params: {
messages: Array<{ role: string; content: string }>;
schema: z.ZodType<T>;
chat?: LlmChatCaller;
models?: string[];
}): Promise<T> {
const chat = params.chat || ollamaChat;
const models = [PRIMARY_MODEL, FALLBACK_MODEL];
const models = params.models?.length ? params.models : [PRIMARY_MODEL, FALLBACK_MODEL];
for (const model of models) {
const msgs = [...params.messages];
@@ -193,7 +195,7 @@ Examples:
},
];
return generateWithValidation({ messages, schema: classificationSchema, chat });
return generateWithValidation({ messages, schema: classificationSchema, chat, models: [INTENT_MODEL, FALLBACK_MODEL] });
}
// ── Info Response Generation ──