zz_

app/agent/ollamaBackend.ts

zzmq.dev@gmail.com/zz / app / agent / ollamaBackend.ts @ b27e1ef0 Download
Ladder Summary
  • file The file implements a **ChatBackend** that talks to a local Ollama server via its `/api/chat` endpoint, streaming the assistant’s response token‑by‑token (and emitting any tool calls once they’re generated) while handling connection‑error details. It converts the generic `ToolDef[]` into Ollama’s function‑tool format with `toOllamaTools` and returns an object exposing `name` and a `chat` method that forwards messages, optional tools, and an optional `onPartial` callback. **Exported symbols** * `createOllamaBackend(model: string, baseUrl: string): ChatBackend` – builds and returns the Ollama‑specific backend.
 // A local Ollama model via /api/chat, which genuinely supports
 // OpenAI-style function-calling for tool-capable models (verified
 // directly against qwen2.5:14b-instruct). Streams the real response:
 // prose content arrives token-by-token, tool_calls arrive as one
 // complete chunk once decided (verified directly against Ollama's own
 // wire format) -- same total generation time as non-streaming, just
 // delivered incrementally instead of buffered until the end.
 import type { ChatBackend, ChatMessage } from "./chatBackend.js";
 import type { ToolDef } from "./tools.js";
 
 function toOllamaTools(tools: ToolDef[]) {
  return tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } }));
 }
 
 export function createOllamaBackend(model: string, baseUrl: string): ChatBackend {
  return {
  name: `ollama:${model}`,
  async chat(messages, tools, onPartial) {
  const res = await fetch(`${baseUrl}/api/chat`, {
  method: "POST",
  headers: { "content-type": "application/json" },
  body: JSON.stringify({ model, messages, ...(tools.length ? { tools: toOllamaTools(tools) } : {}), stream: true }),
  });
  if (!res.ok) throw new Error(`ollama chat failed: ${res.status} ${await res.text()}`);
  if (!res.body) throw new Error("ollama chat response had no body to stream");
 
  let content = "";
  let toolCalls: ChatMessage["tool_calls"];
  let buffer = "";
  for await (const rawChunk of res.body as unknown as AsyncIterable<Uint8Array>) {
  buffer += Buffer.from(rawChunk).toString("utf8");
  let newlineAt: number;
  while ((newlineAt = buffer.indexOf("\n")) >= 0) {
  const line = buffer.slice(0, newlineAt).trim();
  buffer = buffer.slice(newlineAt + 1);
  if (!line) continue;
  const chunk = JSON.parse(line) as { message?: { content?: string; tool_calls?: ChatMessage["tool_calls"] } };
  if (chunk.message?.content) {
  content += chunk.message.content;
  onPartial?.(content);
  }
  if (chunk.message?.tool_calls) toolCalls = chunk.message.tool_calls;
  }
  }
  return { role: "assistant", content, tool_calls: toolCalls };
  },
  };
 }