First cut
This commit is contained in:
kadil
172
server/src/llm/client.ts
Normal file
172
server/src/llm/client.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
import { z } from 'zod';
|
||||
import { routingOutputSchema, type RoutingOutput } from '@dynamic-sites/shared';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
const OLLAMA_HOST = process.env.OLLAMA_HOST || 'https://ollama.com';
|
||||
const OLLAMA_API_KEY = process.env.OLLAMA_API_KEY || '';
|
||||
const PRIMARY_MODEL = process.env.OLLAMA_MODEL || 'qwen3.5:397b-cloud';
|
||||
const FALLBACK_MODEL = process.env.OLLAMA_FALLBACK_MODEL || 'gpt-oss:120b';
|
||||
const MAX_RETRIES = 3;
|
||||
|
||||
export interface LlmChatCaller {
|
||||
(messages: Array<{ role: string; content: string }>, model: string): Promise<string>;
|
||||
}
|
||||
|
||||
/** Default chat caller using Ollama HTTP API */
|
||||
async function ollamaChat(messages: Array<{ role: string; content: string }>, model: string): Promise<string> {
|
||||
const resp = await fetch(`${OLLAMA_HOST}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...(OLLAMA_API_KEY ? { Authorization: `Bearer ${OLLAMA_API_KEY}` } : {}),
|
||||
},
|
||||
body: JSON.stringify({ model, messages, stream: false }),
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
throw new Error(`Ollama ${resp.status}: ${await resp.text().catch(() => 'no body')}`);
|
||||
}
|
||||
|
||||
const data = await resp.json() as { message?: { content?: string } };
|
||||
return data.message?.content || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate-then-retry loop: parse LLM output as JSON, validate against schema,
|
||||
* re-prompt with errors if invalid, up to MAX_RETRIES per model.
|
||||
*/
|
||||
async function generateWithValidation<T>(params: {
|
||||
messages: Array<{ role: string; content: string }>;
|
||||
schema: z.ZodType<T>;
|
||||
chat?: LlmChatCaller;
|
||||
}): Promise<T> {
|
||||
const chat = params.chat || ollamaChat;
|
||||
const models = [PRIMARY_MODEL, FALLBACK_MODEL];
|
||||
|
||||
for (const model of models) {
|
||||
const msgs = [...params.messages];
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
logger.debug({ event: 'llm.request', model, attempt }, 'LLM call');
|
||||
try {
|
||||
const raw = await chat(msgs, model);
|
||||
|
||||
// Extract JSON from response (handle markdown code blocks)
|
||||
const jsonMatch = raw.match(/```(?:json)?\s*([\s\S]*?)```/) || [null, raw];
|
||||
const jsonStr = (jsonMatch[1] || raw).trim();
|
||||
const parsed = JSON.parse(jsonStr);
|
||||
const result = params.schema.safeParse(parsed);
|
||||
|
||||
if (result.success) return result.data;
|
||||
|
||||
const errors = result.error.issues.map(i => `${i.path.join('.')}: ${i.message}`).join('\n');
|
||||
logger.debug({ event: 'llm.retry', model, attempt, errors }, 'Validation failed, retrying');
|
||||
|
||||
msgs.push(
|
||||
{ role: 'assistant', content: raw },
|
||||
{ role: 'user', content: `Your JSON output failed validation:\n${errors}\n\nPlease fix the issues and return valid JSON only.` }
|
||||
);
|
||||
} catch (err) {
|
||||
logger.warn({ event: 'llm.retry', model, attempt, error: (err as Error).message }, 'LLM call or parse failed');
|
||||
if (attempt === MAX_RETRIES - 1) break;
|
||||
msgs.push(
|
||||
{ role: 'user', content: `Your response was not valid JSON. Please respond with ONLY a JSON object, no markdown or extra text.` }
|
||||
);
|
||||
}
|
||||
}
|
||||
logger.warn({ event: 'llm.fallback', from: model }, 'Exhausted retries, trying fallback');
|
||||
}
|
||||
|
||||
logger.error({ event: 'llm.exhausted' }, 'All LLM models exhausted');
|
||||
throw new Error('LLM_UNAVAILABLE');
|
||||
}
|
||||
|
||||
// ── Public API ──
|
||||
|
||||
export interface GenerateEditedJsonParams {
|
||||
currentJson: unknown;
|
||||
siteContext: unknown;
|
||||
userMessage: string;
|
||||
repoRelativePath: string;
|
||||
schema: z.ZodTypeAny;
|
||||
}
|
||||
|
||||
export async function generateEditedJson(params: GenerateEditedJsonParams, chat?: LlmChatCaller): Promise<unknown> {
|
||||
const messages = [
|
||||
{
|
||||
role: 'system',
|
||||
content: `You are a website content editor. You edit JSON content files for a website.
|
||||
|
||||
SITE CONTEXT:
|
||||
${JSON.stringify(params.siteContext, null, 2)}
|
||||
|
||||
You will receive the current JSON content of a section file and a natural language edit request.
|
||||
Return ONLY the complete updated JSON object — no explanation, no markdown, just the JSON.
|
||||
Preserve all existing fields and structure. Only change what the user requested.
|
||||
The output must be valid JSON matching the exact same schema as the input.`,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `Current content of "${params.repoRelativePath}":\n\`\`\`json\n${JSON.stringify(params.currentJson, null, 2)}\n\`\`\`\n\nEdit request: "${params.userMessage}"\n\nReturn the complete updated JSON:`,
|
||||
},
|
||||
];
|
||||
|
||||
return generateWithValidation({ messages, schema: params.schema, chat });
|
||||
}
|
||||
|
||||
export interface RouteEditIntentParams {
|
||||
userMessage: string;
|
||||
manifest: Array<{ id: string; type: string; title?: string; headline?: string; heading?: string; repo_relative_path: string; visible: boolean }>;
|
||||
}
|
||||
|
||||
export async function routeEditIntent(params: RouteEditIntentParams, chat?: LlmChatCaller): Promise<RoutingOutput> {
|
||||
const messages = [
|
||||
{
|
||||
role: 'system',
|
||||
content: `You are a routing assistant for a website CMS. Given a natural language edit request and a manifest of available content sections, determine which section file the edit applies to.
|
||||
|
||||
Return a JSON object with:
|
||||
- "repo_relative_path": the path of the target section file
|
||||
- "needs_clarification": true if the request is ambiguous
|
||||
- "reason": short explanation
|
||||
- "clarification_message": (only if needs_clarification) a question to ask the user
|
||||
|
||||
If the request is about showing/hiding/enabling/disabling a section, route to that section's file.
|
||||
If the request mentions events, route to "content/events.json".`,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `MANIFEST:\n${JSON.stringify(params.manifest, null, 2)}\n\nEDIT REQUEST: "${params.userMessage}"\n\nReturn JSON:`,
|
||||
},
|
||||
];
|
||||
|
||||
return generateWithValidation({ messages, schema: routingOutputSchema, chat });
|
||||
}
|
||||
|
||||
/** Simple summary generation (no schema validation needed) */
|
||||
export async function generateSummary(params: {
|
||||
before: unknown;
|
||||
after: unknown;
|
||||
repoRelativePath: string;
|
||||
userMessage: string;
|
||||
chat?: LlmChatCaller;
|
||||
}): Promise<string> {
|
||||
const chat = params.chat || ollamaChat;
|
||||
const messages = [
|
||||
{
|
||||
role: 'system',
|
||||
content: `You summarize content changes for a website owner. Keep summaries under 140 characters, plain text, no markdown. Be specific about what changed. Format: "Change X from A to B" or "Add/remove X".`,
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `File: ${params.repoRelativePath}\nRequest: "${params.userMessage}"\n\nBefore:\n${JSON.stringify(params.before, null, 2)}\n\nAfter:\n${JSON.stringify(params.after, null, 2)}\n\nSummarize the change in under 140 chars:`,
|
||||
},
|
||||
];
|
||||
|
||||
try {
|
||||
const result = await chat(messages, PRIMARY_MODEL);
|
||||
return result.replace(/["'`]/g, '').trim().slice(0, 280);
|
||||
} catch {
|
||||
// Fallback: generate a basic diff summary
|
||||
return `Update ${params.repoRelativePath} as requested: "${params.userMessage.slice(0, 80)}"`;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user