Document Ollama inference setup

This commit is contained in:
mmckeen-nv
2026-05-20 18:50:24 -07:00
parent 9d0e0e1a16
commit 81f7239bf3
7 changed files with 499 additions and 50 deletions
+285
View File
@@ -0,0 +1,285 @@
#!/usr/bin/env node
import http from "node:http";
const listenHost = process.env.OLLAMA_PROXY_HOST || "0.0.0.0";
const listenPort = Number.parseInt(process.env.OLLAMA_PROXY_PORT || "11435", 10);
const upstream = (process.env.OLLAMA_PROXY_UPSTREAM || "http://172.18.16.1:11434").replace(/\/+$/, "");
const numCtx = Number.parseInt(process.env.OLLAMA_PROXY_NUM_CTX || "8192", 10);
function sendJson(res, status, payload) {
const body = JSON.stringify(payload);
res.writeHead(status, {
"content-type": "application/json",
"content-length": Buffer.byteLength(body),
});
res.end(body);
}
async function readJson(req) {
let body = "";
for await (const chunk of req) body += String(chunk);
return body.trim() ? JSON.parse(body) : {};
}
function normalizeMessages(messages) {
return Array.isArray(messages)
? messages.map((message) => ({
role: String(message?.role || "user"),
content: Array.isArray(message?.content)
? message.content.map((part) => part?.text || "").join("\n")
: String(message?.content ?? ""),
}))
: [];
}
function finishReason(doneReason) {
if (doneReason === "stop") return "stop";
if (doneReason === "length") return "length";
return doneReason || "stop";
}
function sendSse(res, payloads) {
res.writeHead(200, {
"content-type": "text/event-stream; charset=utf-8",
"cache-control": "no-cache",
connection: "keep-alive",
});
for (const payload of payloads) {
res.write(`data: ${JSON.stringify(payload)}\n\n`);
}
res.write("data: [DONE]\n\n");
res.end();
}
function writeSse(res, payload) {
res.write(`data: ${JSON.stringify(payload)}\n\n`);
}
async function proxyModels(res) {
const response = await fetch(`${upstream}/v1/models`);
const text = await response.text();
res.writeHead(response.status, { "content-type": response.headers.get("content-type") || "application/json" });
res.end(text);
}
async function proxyStreamingChat(res, model, nativeBody) {
const id = `chatcmpl-${Date.now().toString(36)}`;
const created = Math.floor(Date.now() / 1000);
res.writeHead(200, {
"content-type": "text/event-stream; charset=utf-8",
"cache-control": "no-cache",
connection: "keep-alive",
});
writeSse(res, {
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
});
const response = await fetch(`${upstream}/api/chat`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ ...nativeBody, stream: true }),
});
if (!response.ok) {
const text = await response.text();
writeSse(res, {
id,
object: "chat.completion.chunk",
created,
model,
choices: [
{
index: 0,
delta: { content: "" },
finish_reason: "stop",
},
],
error: {
message: text || `Ollama upstream returned ${response.status}`,
type: "upstream_error",
},
});
res.write("data: [DONE]\n\n");
res.end();
return;
}
const decoder = new TextDecoder();
let buffer = "";
let finish = "stop";
let sentContent = false;
for await (const chunk of response.body) {
buffer += decoder.decode(chunk, { stream: true });
const lines = buffer.split(/\r?\n/);
buffer = lines.pop() || "";
for (const line of lines) {
if (!line.trim()) continue;
const payload = JSON.parse(line);
const content = String(payload?.message?.content ?? payload?.response ?? "");
if (content) {
sentContent = true;
writeSse(res, {
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { content }, finish_reason: null }],
});
}
if (payload?.done) finish = finishReason(payload?.done_reason);
}
}
if (buffer.trim()) {
const payload = JSON.parse(buffer);
const content = String(payload?.message?.content ?? payload?.response ?? "");
if (content) {
sentContent = true;
writeSse(res, {
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { content }, finish_reason: null }],
});
}
if (payload?.done) finish = finishReason(payload?.done_reason);
}
if (!sentContent) {
writeSse(res, {
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { content: "" }, finish_reason: null }],
});
}
writeSse(res, {
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: {}, finish_reason: finish }],
});
res.write("data: [DONE]\n\n");
res.end();
}
async function proxyChat(req, res) {
const body = await readJson(req);
const model = String(body.model || process.env.OLLAMA_PROXY_MODEL || "qwen3.5:27b");
const nativeBody = {
model,
messages: normalizeMessages(body.messages),
stream: false,
think: false,
options: {
temperature: body.temperature ?? 0,
num_predict: body.max_tokens ?? body.max_completion_tokens ?? 512,
...(Number.isFinite(numCtx) && numCtx > 0 ? { num_ctx: numCtx } : {}),
},
};
if (body.stream === true) {
await proxyStreamingChat(res, model, nativeBody);
return;
}
const response = await fetch(`${upstream}/api/chat`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(nativeBody),
});
const payload = await response.json().catch(async () => ({ error: await response.text() }));
if (!response.ok) {
sendJson(res, response.status, {
error: {
message: payload?.error || `Ollama upstream returned ${response.status}`,
type: "upstream_error",
},
});
return;
}
const content = String(payload?.message?.content ?? payload?.response ?? "");
const completion = {
id: `chatcmpl-${Date.now().toString(36)}`,
object: "chat.completion",
created: Math.floor(Date.now() / 1000),
model,
system_fingerprint: "fp_ollama_native_think_false",
choices: [
{
index: 0,
message: { role: "assistant", content },
finish_reason: finishReason(payload?.done_reason),
},
],
usage: {
prompt_tokens: payload?.prompt_eval_count ?? 0,
completion_tokens: payload?.eval_count ?? 0,
total_tokens: (payload?.prompt_eval_count ?? 0) + (payload?.eval_count ?? 0),
},
};
if (body.stream === true) {
const id = completion.id;
const created = completion.created;
sendSse(res, [
{
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
},
{
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: { content }, finish_reason: null }],
},
{
id,
object: "chat.completion.chunk",
created,
model,
choices: [{ index: 0, delta: {}, finish_reason: completion.choices[0].finish_reason }],
},
]);
return;
}
sendJson(res, 200, completion);
}
const server = http.createServer((req, res) => {
const url = new URL(req.url || "/", `http://${req.headers.host || "localhost"}`);
Promise.resolve()
.then(async () => {
if (req.method === "GET" && (url.pathname === "/v1/models" || url.pathname === "/models")) {
await proxyModels(res);
return;
}
if (req.method === "POST" && (url.pathname === "/v1/chat/completions" || url.pathname === "/chat/completions")) {
await proxyChat(req, res);
return;
}
if (req.method === "GET" && url.pathname === "/health") {
sendJson(res, 200, { ok: true, upstream });
return;
}
sendJson(res, 404, { error: { message: "not found", type: "not_found" } });
})
.catch((error) => {
sendJson(res, 500, { error: { message: error?.message || String(error), type: "proxy_error" } });
});
});
server.listen(listenPort, listenHost, () => {
console.log(`Ollama thinkless OpenAI proxy listening on http://${listenHost}:${listenPort}/v1 -> ${upstream}`);
});