#!/usr/bin/env node import http from "node:http"; const listenHost = process.env.OLLAMA_PROXY_HOST || "0.0.0.0"; const listenPort = Number.parseInt(process.env.OLLAMA_PROXY_PORT || "11435", 10); const upstream = (process.env.OLLAMA_PROXY_UPSTREAM || "http://172.18.16.1:11434").replace(/\/+$/, ""); const numCtx = Number.parseInt(process.env.OLLAMA_PROXY_NUM_CTX || "8192", 10); function sendJson(res, status, payload) { const body = JSON.stringify(payload); res.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(body), }); res.end(body); } async function readJson(req) { let body = ""; for await (const chunk of req) body += String(chunk); return body.trim() ? JSON.parse(body) : {}; } function normalizeMessages(messages) { return Array.isArray(messages) ? messages.map((message) => ({ role: String(message?.role || "user"), content: Array.isArray(message?.content) ? message.content.map((part) => part?.text || "").join("\n") : String(message?.content ?? ""), })) : []; } function finishReason(doneReason) { if (doneReason === "stop") return "stop"; if (doneReason === "length") return "length"; return doneReason || "stop"; } function sendSse(res, payloads) { res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache", connection: "keep-alive", }); for (const payload of payloads) { res.write(`data: ${JSON.stringify(payload)}\n\n`); } res.write("data: [DONE]\n\n"); res.end(); } function writeSse(res, payload) { res.write(`data: ${JSON.stringify(payload)}\n\n`); } async function proxyModels(res) { const response = await fetch(`${upstream}/v1/models`); const text = await response.text(); res.writeHead(response.status, { "content-type": response.headers.get("content-type") || "application/json" }); res.end(text); } async function proxyStreamingChat(res, model, nativeBody) { const id = `chatcmpl-${Date.now().toString(36)}`; const created = Math.floor(Date.now() / 1000); res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache", connection: "keep-alive", }); writeSse(res, { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], }); const response = await fetch(`${upstream}/api/chat`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ ...nativeBody, stream: true }), }); if (!response.ok) { const text = await response.text(); writeSse(res, { id, object: "chat.completion.chunk", created, model, choices: [ { index: 0, delta: { content: "" }, finish_reason: "stop", }, ], error: { message: text || `Ollama upstream returned ${response.status}`, type: "upstream_error", }, }); res.write("data: [DONE]\n\n"); res.end(); return; } const decoder = new TextDecoder(); let buffer = ""; let finish = "stop"; let sentContent = false; for await (const chunk of response.body) { buffer += decoder.decode(chunk, { stream: true }); const lines = buffer.split(/\r?\n/); buffer = lines.pop() || ""; for (const line of lines) { if (!line.trim()) continue; const payload = JSON.parse(line); const content = String(payload?.message?.content ?? payload?.response ?? ""); if (content) { sentContent = true; writeSse(res, { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: { content }, finish_reason: null }], }); } if (payload?.done) finish = finishReason(payload?.done_reason); } } if (buffer.trim()) { const payload = JSON.parse(buffer); const content = String(payload?.message?.content ?? payload?.response ?? ""); if (content) { sentContent = true; writeSse(res, { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: { content }, finish_reason: null }], }); } if (payload?.done) finish = finishReason(payload?.done_reason); } if (!sentContent) { writeSse(res, { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: { content: "" }, finish_reason: null }], }); } writeSse(res, { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: {}, finish_reason: finish }], }); res.write("data: [DONE]\n\n"); res.end(); } async function proxyChat(req, res) { const body = await readJson(req); const model = String(body.model || process.env.OLLAMA_PROXY_MODEL || "qwen3.5:27b"); const nativeBody = { model, messages: normalizeMessages(body.messages), stream: false, think: false, options: { temperature: body.temperature ?? 0, num_predict: body.max_tokens ?? body.max_completion_tokens ?? 512, ...(Number.isFinite(numCtx) && numCtx > 0 ? { num_ctx: numCtx } : {}), }, }; if (body.stream === true) { await proxyStreamingChat(res, model, nativeBody); return; } const response = await fetch(`${upstream}/api/chat`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(nativeBody), }); const payload = await response.json().catch(async () => ({ error: await response.text() })); if (!response.ok) { sendJson(res, response.status, { error: { message: payload?.error || `Ollama upstream returned ${response.status}`, type: "upstream_error", }, }); return; } const content = String(payload?.message?.content ?? payload?.response ?? ""); const completion = { id: `chatcmpl-${Date.now().toString(36)}`, object: "chat.completion", created: Math.floor(Date.now() / 1000), model, system_fingerprint: "fp_ollama_native_think_false", choices: [ { index: 0, message: { role: "assistant", content }, finish_reason: finishReason(payload?.done_reason), }, ], usage: { prompt_tokens: payload?.prompt_eval_count ?? 0, completion_tokens: payload?.eval_count ?? 0, total_tokens: (payload?.prompt_eval_count ?? 0) + (payload?.eval_count ?? 0), }, }; if (body.stream === true) { const id = completion.id; const created = completion.created; sendSse(res, [ { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], }, { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: { content }, finish_reason: null }], }, { id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta: {}, finish_reason: completion.choices[0].finish_reason }], }, ]); return; } sendJson(res, 200, completion); } const server = http.createServer((req, res) => { const url = new URL(req.url || "/", `http://${req.headers.host || "localhost"}`); Promise.resolve() .then(async () => { if (req.method === "GET" && (url.pathname === "/v1/models" || url.pathname === "/models")) { await proxyModels(res); return; } if (req.method === "POST" && (url.pathname === "/v1/chat/completions" || url.pathname === "/chat/completions")) { await proxyChat(req, res); return; } if (req.method === "GET" && url.pathname === "/health") { sendJson(res, 200, { ok: true, upstream }); return; } sendJson(res, 404, { error: { message: "not found", type: "not_found" } }); }) .catch((error) => { sendJson(res, 500, { error: { message: error?.message || String(error), type: "proxy_error" } }); }); }); server.listen(listenPort, listenHost, () => { console.log(`Ollama thinkless OpenAI proxy listening on http://${listenHost}:${listenPort}/v1 -> ${upstream}`); });