From 81f7239bf3790fc6881f0e394cd2811197523ba9 Mon Sep 17 00:00:00 2001 From: mmckeen-nv Date: Wed, 20 May 2026 18:49:44 -0700 Subject: [PATCH] Document Ollama inference setup --- .env.example | 8 + OLLAMA_INFERENCE.md | 118 +++++++++ QUICKSTART.md | 84 +++---- README.md | 16 ++ app/lib/sandboxInferenceApply.ts | 26 +- install.sh | 12 +- scripts/ollama-thinkless-openai-proxy.mjs | 285 ++++++++++++++++++++++ 7 files changed, 499 insertions(+), 50 deletions(-) create mode 100644 OLLAMA_INFERENCE.md create mode 100644 scripts/ollama-thinkless-openai-proxy.mjs diff --git a/.env.example b/.env.example index 9109959..812cf82 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,14 @@ OPENSHELL_CONTROL_CREATE_GPU_MODE=none # OPENSHELL_GATEWAY_PORT=8080 # OPENSHELL_GATEWAY_URL=http://host.docker.internal:8080 +# Ollama discovery and inference routing. +# In WSL2, prefer a WSL-local Ollama server for large models. The model picker also +# probes the Windows host gateway and Windows localhost through WSL interop. +OPENSHELL_OLLAMA_BASE_URL=http://127.0.0.1:11434 +# OPENSHELL_OLLAMA_HOSTS=127.0.0.1,172.19.0.1 +# OPENSHELL_OLLAMA_PROBE_TIMEOUT_MS=2500 +OPENSHELL_OLLAMA_WINDOWS_INTEROP=1 + # Dashboard port. PORT controls the server listener; NEXT_PUBLIC_DASHBOARD_PORT is exposed to the UI. PORT=3000 NEXT_PUBLIC_DASHBOARD_PORT=3000 diff --git a/OLLAMA_INFERENCE.md b/OLLAMA_INFERENCE.md new file mode 100644 index 0000000..0bf4ae0 --- /dev/null +++ b/OLLAMA_INFERENCE.md @@ -0,0 +1,118 @@ +# Ollama Inference Runbook + +This dashboard can route sandbox inference through OpenShell's `inference.local` +gateway to an OpenAI-compatible Ollama endpoint. + +## Recommended WSL2 Path + +For large local models, run Ollama inside WSL2 and point the OpenShell provider at +WSL localhost: + +```bash +ollama serve +ollama pull nemotron-3-super:120b +ollama show nemotron-3-super:120b + +openshell provider create \ + --name ollama-local \ + --type openai \ + --credential OPENAI_API_KEY=ollama \ + --config OPENAI_BASE_URL=http://127.0.0.1:11434/v1 + +# If the provider already exists: +openshell provider update ollama-local \ + --credential OPENAI_API_KEY=ollama \ + --config OPENAI_BASE_URL=http://127.0.0.1:11434/v1 + +openshell inference set \ + --no-verify \ + --provider ollama-local \ + --model nemotron-3-super:120b \ + --timeout 600 +``` + +Use `openshell inference get` to verify the active route. + +## Validated Local Models + +These values are what Ollama reported on the WSL2 test host. + +| Model | Size | Ollama context | Notes | +| --- | ---: | ---: | --- | +| `qwen2.5:7b` | 4.7 GB | 32768 | Fast fallback; enough for short cuFolio card requests after session reset. | +| `nemotron-3-super:120b` | 86 GB | 262144 | Fits on a 98 GB RTX 6000 Blackwell at Q4_K_M; much slower, but handles the cuFolio/OpenClaw prompt with far more context headroom. | + +The `nemotron-3-super:120b` smoke test loaded entirely into RTX 6000 VRAM: + +```text +size_vram: 94227902208 +context_length: 262144 +``` + +The end-to-end cuFolio sandbox chat test completed successfully with: + +```text +model: inference/nemotron-3-super:120b +pending: false +answer: exact requested token +elapsed: about 91 seconds +``` + +## OpenClaw Context Behavior + +OpenClaw reserves output/recovery tokens before building the prompt. With a 32768 +token model and a 20000 token reserve, only about 12768 tokens remain for input +history. That is why long-running `inter-sandbox-chat` sessions can overflow even +when Ollama reports a 32k model context. + +The sandbox inference live-apply path now publishes model metadata for known +Ollama models: + +- `qwen2.5:7b`: `contextWindow=32768`, `maxTokens=2048` +- `qwen3.5:27b`: `contextWindow=32768`, `maxTokens=2048` +- `nemotron-3-super:120b`: `contextWindow=262144`, `maxTokens=8192` + +If a sandbox has accumulated a very large chat transcript, reset or compact the +OpenClaw session before judging model health. In the cuFolio smoke test, an old +`inter-sandbox-chat` transcript caused a pre-inference context overflow; after +OpenClaw reset that session, the same WSL2 Ollama path answered correctly. + +## Windows-Host Ollama Notes + +Windows-host Ollama can be selected during onboarding and model discovery. From +WSL2, the dashboard probes: + +- WSL localhost: `http://127.0.0.1:11434` +- WSL Windows-host gateway candidates from `/etc/resolv.conf` and `/proc/net/route` +- `host.docker.internal` +- Windows localhost through `curl.exe` / PowerShell interop + +Useful environment overrides: + +```bash +OPENSHELL_OLLAMA_BASE_URL=http://127.0.0.1:11434 +OPENSHELL_OLLAMA_HOSTS=127.0.0.1,172.19.0.1 +OPENSHELL_OLLAMA_PROBE_TIMEOUT_MS=2500 +OPENSHELL_OLLAMA_WINDOWS_INTEROP=1 +OPENSHELL_WINDOWS_OLLAMA_BASE_URL=http://127.0.0.1:11434 +``` + +During testing, Windows-host `qwen3.5:27b` could expose reasoning in the +OpenAI-compatible response while leaving `message.content` empty. The optional +proxy below converts OpenAI chat requests to Ollama native `/api/chat` with +`think:false` and streams early SSE events to avoid OpenClaw idle timeouts: + +```bash +OLLAMA_PROXY_HOST=0.0.0.0 \ +OLLAMA_PROXY_PORT=11435 \ +OLLAMA_PROXY_UPSTREAM=http://172.18.16.1:11434 \ +OLLAMA_PROXY_NUM_CTX=32768 \ +node scripts/ollama-thinkless-openai-proxy.mjs + +openshell provider update ollama-win \ + --credential OPENAI_API_KEY=ollama \ + --config OPENAI_BASE_URL=http://172.19.0.1:11435/v1 +``` + +Prefer the WSL2-local path for the cuFolio demo unless the Windows-host model is +known to return normal OpenAI `message.content`. diff --git a/QUICKSTART.md b/QUICKSTART.md index 3ad3192..ee9ec3d 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -4,12 +4,12 @@ 1. **Navigate to project directory:** ```bash -cd /path/to/nemoclaw-dashboard +cd /path/to/openshell_controller ``` 2. **Install dependencies:** ```bash -npm install +./install.sh ``` 3. **Start the development server:** @@ -24,40 +24,38 @@ http://localhost:3000 ## What You'll See -- **Configuration Panel** at the top - set your OpenShell and Ollama URLs -- **Telemetry Cards** - CPU, memory, disk, and GPU usage (updates every 3 seconds) -- **One-Click Actions** - Start Sandbox, Stop Sandbox, Create Instance, Switch Model buttons -- **Model Switcher** - Dropdown to select Ollama models -- **Configuration Panels** - View settings for OpenShell, NemoClaw, and Ollama +- **Sandbox inventory** from the OpenShell gateway +- **Operator terminal** and OpenClaw dashboard proxy links +- **Inference settings** for OpenAI-compatible providers, Ollama, NIM, and vLLM +- **Ollama model discovery** across WSL2 localhost and optional Windows-host Ollama +- **MCP broker settings** and per-sandbox MCP access controls -## Testing the Dashboard +## Ollama Smoke Test -### Test 1: Verify Dashboard Loads -1. Open `http://localhost:3000` -2. All sections should render without errors -3. Telemetry should show values (randomly generated) +For WSL2 demo hosts, run Ollama in WSL2: -### Test 2: Configuration Panel -1. Edit the OpenShell URL field -2. Edit the Ollama URL field -3. Settings are saved to localStorage (persists on refresh) +```bash +ollama serve +ollama pull qwen2.5:7b +ollama pull nemotron-3-super:120b +``` -### Test 3: One-Click Actions -1. Click "Start Sandbox" -2. Click "Stop Sandbox" -3. Click "Create Instance" -4. Click "Switch Model" -5. Check that status messages appear +Configure the gateway: -### Test 4: Model Switcher -1. Select a different model from the dropdown -2. Click "Switch Model" -3. Verify success message +```bash +openshell provider update ollama-local \ + --credential OPENAI_API_KEY=ollama \ + --config OPENAI_BASE_URL=http://127.0.0.1:11434/v1 -### Test 5: Telemetry Updates -1. Wait 3 seconds -2. Telemetry values should update automatically -3. No console errors should appear +openshell inference set \ + --no-verify \ + --provider ollama-local \ + --model nemotron-3-super:120b \ + --timeout 600 +``` + +`qwen2.5:7b` reports a 32768 token context. `nemotron-3-super:120b` reports a +262144 token context and needs about 94 GB of RTX 6000 VRAM at Q4_K_M. ## Troubleshooting @@ -67,27 +65,21 @@ http://localhost:3000 - Restart the server ### API endpoints returning errors -- This is expected - endpoints are mocked for development -- Check browser console for specific errors -- No real connection to OpenShell/Ollama is required +- Check `openshell sandbox list` +- Check that the OpenShell gateway container is running +- Check browser console and server logs for specific errors + +### Ollama models do not appear +- Confirm WSL2 Ollama is reachable: `curl http://127.0.0.1:11434/api/tags` +- If using Windows-host Ollama, confirm it is reachable from WSL2 through the + detected host gateway or Windows interop +- Review `OLLAMA_INFERENCE.md` for the supported environment overrides ### Dependencies not installing -- Make sure you're running Node.js 18 or later +- Make sure you're running Node.js 20 or later - Clear npm cache: `npm cache clean --force` - Try reinstalling: `rm -rf node_modules package-lock.json && npm install` -## Next Steps (Production) - -To make this production-ready: - -1. Replace mock API routes with real API calls to OpenShell (port 8080) and Ollama (port 11434) -2. Add authentication to API routes -3. Implement proper error handling -4. Add loading states and skeletons -5. Add dark mode toggle -6. Add more detailed telemetry charts -7. Add deployment configuration (Docker, Vercel, etc.) - ## Support For issues or questions: diff --git a/README.md b/README.md index 57578a2..90ab03e 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,22 @@ Default ports: The dashboard WebSocket proxy is served on the same listener by default, using `/api/openshell/dashboard/proxy` or `/api/openshell/instances/[instanceId]/dashboard/proxy`. Set `OPENCLAW_DASHBOARD_WS_PROXY_PORT=3001` only if you intentionally want the legacy dedicated sidecar listener. +## Ollama Inference + +For WSL2 demo hosts, prefer running Ollama inside WSL2 and routing OpenShell +through `ollama-local` at `http://127.0.0.1:11434/v1`. This avoids Windows/WSL +host routing surprises and was validated with both `qwen2.5:7b` and +`nemotron-3-super:120b`. + +The `nemotron-3-super:120b` Ollama tag reports a `262144` token context window +and was smoke-tested end to end through cuFolio sandbox chat on an RTX 6000 +Blackwell. It fit in VRAM at Q4_K_M, but responses are much slower than the 7B +fallback. + +See [OLLAMA_INFERENCE.md](OLLAMA_INFERENCE.md) for setup commands, context-window +notes, and the optional Windows-host thinkless proxy used for models that return +reasoning without OpenAI `message.content`. + ## Authentication This project currently uses a simple local password and signed HTTP-only cookie. diff --git a/app/lib/sandboxInferenceApply.ts b/app/lib/sandboxInferenceApply.ts index ebf205f..9aeacf3 100644 --- a/app/lib/sandboxInferenceApply.ts +++ b/app/lib/sandboxInferenceApply.ts @@ -7,16 +7,36 @@ const execFileAsync = promisify(execFile) const DOCKER_BIN = process.env.DOCKER_BIN || "docker" const OPENSHELL_CLUSTER_CONTAINER = process.env.OPENSHELL_CLUSTER_CONTAINER || "openshell-cluster-nemoclaw" +function modelContextWindow(modelId: string) { + const normalized = modelId.toLowerCase() + if (normalized.includes("nemotron-3-super") && normalized.includes("120b")) return 262144 + if (normalized.includes("qwen2.5:7b")) return 32768 + if (normalized.includes("qwen3.5:27b")) return 32768 + return 131072 +} + +function modelMaxTokens(modelId: string) { + const contextWindow = modelContextWindow(modelId) + if (contextWindow >= 262144) return 8192 + if (contextWindow <= 32768) return 2048 + return 4096 +} + +function modelSupportsReasoning(modelId: string) { + const normalized = modelId.toLowerCase() + return normalized.includes("nemotron-3-super") +} + function modelEntry(modelId: string, modelName: string, compat: Record | null) { return { ...(compat ? { compat } : {}), id: modelId, name: modelName, - reasoning: false, + reasoning: modelSupportsReasoning(modelId), input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 131072, - maxTokens: 4096, + contextWindow: modelContextWindow(modelId), + maxTokens: modelMaxTokens(modelId), } } diff --git a/install.sh b/install.sh index a12a9fc..34cd298 100755 --- a/install.sh +++ b/install.sh @@ -299,7 +299,7 @@ upsert_env() { set_env() { local key="$1" local value="$2" - [[ -n "$value" ]] || return + [[ -n "$value" ]] || return 0 touch "$ENV_FILE" if grep -qE "^${key}=" "$ENV_FILE"; then sed -i.bak -E "s|^${key}=.*$|${key}=${value}|" "$ENV_FILE" @@ -408,6 +408,12 @@ OPENSHELL_CONTROL_CREATE_GPU_MODE=none # OPENSHELL_GATEWAY_HOST=host.docker.internal # OPENSHELL_GATEWAY_PORT=8080 # OPENSHELL_GATEWAY_URL=http://host.docker.internal:8080 +# Ollama model discovery prefers WSL2 localhost, then the Windows host gateway. +# Keep Ollama in WSL2 for large local models unless you intentionally configure Windows-host Ollama. +# OPENSHELL_OLLAMA_BASE_URL=http://127.0.0.1:11434 +# OPENSHELL_OLLAMA_HOSTS=127.0.0.1,172.19.0.1 +# OPENSHELL_OLLAMA_PROBE_TIMEOUT_MS=2500 +# OPENSHELL_OLLAMA_WINDOWS_INTEROP=1 # For reverse proxies, OpenClaw dashboard websockets use the same-origin proxy path by default. # OPENCLAW_DASHBOARD_BASE_WS_URL=wss://control.example.com # BASE_WS_URL=wss://control.example.com @@ -424,9 +430,13 @@ upsert_env "NEXT_PUBLIC_ENABLE_SANDBOX_OPERATIONS" "true" upsert_env "OPEN_SHELL_CONTAINER" "$OPEN_SHELL_CONTAINER_DEFAULT" upsert_env "OPENSHELL_GATEWAY" "nemoclaw" upsert_env "OPENSHELL_CONTROL_CREATE_GPU_MODE" "none" +upsert_env "OPENSHELL_OLLAMA_BASE_URL" "http://127.0.0.1:11434" +upsert_env "OPENSHELL_OLLAMA_WINDOWS_INTEROP" "1" set_env "OPENSHELL_GATEWAY_HOST" "${OPENSHELL_GATEWAY_HOST:-}" set_env "OPENSHELL_GATEWAY_PORT" "${OPENSHELL_GATEWAY_PORT:-}" set_env "OPENSHELL_GATEWAY_URL" "${OPENSHELL_GATEWAY_URL:-}" +set_env "OPENSHELL_OLLAMA_HOSTS" "${OPENSHELL_OLLAMA_HOSTS:-}" +set_env "OPENSHELL_OLLAMA_PROBE_TIMEOUT_MS" "${OPENSHELL_OLLAMA_PROBE_TIMEOUT_MS:-}" set_env "OPENSHELL_CONTROL_VENV" "${VIRTUAL_ENV:-$PROJECT_VENV}" set_env "OPENSHELL_HOME" "${HOME:-}" set_env "OPENSHELL_BIN" "${OPENSHELL_BIN:-}" diff --git a/scripts/ollama-thinkless-openai-proxy.mjs b/scripts/ollama-thinkless-openai-proxy.mjs new file mode 100644 index 0000000..e02d644 --- /dev/null +++ b/scripts/ollama-thinkless-openai-proxy.mjs @@ -0,0 +1,285 @@ +#!/usr/bin/env node +import http from "node:http"; + +const listenHost = process.env.OLLAMA_PROXY_HOST || "0.0.0.0"; +const listenPort = Number.parseInt(process.env.OLLAMA_PROXY_PORT || "11435", 10); +const upstream = (process.env.OLLAMA_PROXY_UPSTREAM || "http://172.18.16.1:11434").replace(/\/+$/, ""); +const numCtx = Number.parseInt(process.env.OLLAMA_PROXY_NUM_CTX || "8192", 10); + +function sendJson(res, status, payload) { + const body = JSON.stringify(payload); + res.writeHead(status, { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + }); + res.end(body); +} + +async function readJson(req) { + let body = ""; + for await (const chunk of req) body += String(chunk); + return body.trim() ? JSON.parse(body) : {}; +} + +function normalizeMessages(messages) { + return Array.isArray(messages) + ? messages.map((message) => ({ + role: String(message?.role || "user"), + content: Array.isArray(message?.content) + ? message.content.map((part) => part?.text || "").join("\n") + : String(message?.content ?? ""), + })) + : []; +} + +function finishReason(doneReason) { + if (doneReason === "stop") return "stop"; + if (doneReason === "length") return "length"; + return doneReason || "stop"; +} + +function sendSse(res, payloads) { + res.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache", + connection: "keep-alive", + }); + for (const payload of payloads) { + res.write(`data: ${JSON.stringify(payload)}\n\n`); + } + res.write("data: [DONE]\n\n"); + res.end(); +} + +function writeSse(res, payload) { + res.write(`data: ${JSON.stringify(payload)}\n\n`); +} + +async function proxyModels(res) { + const response = await fetch(`${upstream}/v1/models`); + const text = await response.text(); + res.writeHead(response.status, { "content-type": response.headers.get("content-type") || "application/json" }); + res.end(text); +} + +async function proxyStreamingChat(res, model, nativeBody) { + const id = `chatcmpl-${Date.now().toString(36)}`; + const created = Math.floor(Date.now() / 1000); + res.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache", + connection: "keep-alive", + }); + writeSse(res, { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }); + + const response = await fetch(`${upstream}/api/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ...nativeBody, stream: true }), + }); + + if (!response.ok) { + const text = await response.text(); + writeSse(res, { + id, + object: "chat.completion.chunk", + created, + model, + choices: [ + { + index: 0, + delta: { content: "" }, + finish_reason: "stop", + }, + ], + error: { + message: text || `Ollama upstream returned ${response.status}`, + type: "upstream_error", + }, + }); + res.write("data: [DONE]\n\n"); + res.end(); + return; + } + + const decoder = new TextDecoder(); + let buffer = ""; + let finish = "stop"; + let sentContent = false; + + for await (const chunk of response.body) { + buffer += decoder.decode(chunk, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() || ""; + for (const line of lines) { + if (!line.trim()) continue; + const payload = JSON.parse(line); + const content = String(payload?.message?.content ?? payload?.response ?? ""); + if (content) { + sentContent = true; + writeSse(res, { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { content }, finish_reason: null }], + }); + } + if (payload?.done) finish = finishReason(payload?.done_reason); + } + } + + if (buffer.trim()) { + const payload = JSON.parse(buffer); + const content = String(payload?.message?.content ?? payload?.response ?? ""); + if (content) { + sentContent = true; + writeSse(res, { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { content }, finish_reason: null }], + }); + } + if (payload?.done) finish = finishReason(payload?.done_reason); + } + + if (!sentContent) { + writeSse(res, { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { content: "" }, finish_reason: null }], + }); + } + writeSse(res, { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: {}, finish_reason: finish }], + }); + res.write("data: [DONE]\n\n"); + res.end(); +} + +async function proxyChat(req, res) { + const body = await readJson(req); + const model = String(body.model || process.env.OLLAMA_PROXY_MODEL || "qwen3.5:27b"); + const nativeBody = { + model, + messages: normalizeMessages(body.messages), + stream: false, + think: false, + options: { + temperature: body.temperature ?? 0, + num_predict: body.max_tokens ?? body.max_completion_tokens ?? 512, + ...(Number.isFinite(numCtx) && numCtx > 0 ? { num_ctx: numCtx } : {}), + }, + }; + + if (body.stream === true) { + await proxyStreamingChat(res, model, nativeBody); + return; + } + + const response = await fetch(`${upstream}/api/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(nativeBody), + }); + const payload = await response.json().catch(async () => ({ error: await response.text() })); + if (!response.ok) { + sendJson(res, response.status, { + error: { + message: payload?.error || `Ollama upstream returned ${response.status}`, + type: "upstream_error", + }, + }); + return; + } + const content = String(payload?.message?.content ?? payload?.response ?? ""); + const completion = { + id: `chatcmpl-${Date.now().toString(36)}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model, + system_fingerprint: "fp_ollama_native_think_false", + choices: [ + { + index: 0, + message: { role: "assistant", content }, + finish_reason: finishReason(payload?.done_reason), + }, + ], + usage: { + prompt_tokens: payload?.prompt_eval_count ?? 0, + completion_tokens: payload?.eval_count ?? 0, + total_tokens: (payload?.prompt_eval_count ?? 0) + (payload?.eval_count ?? 0), + }, + }; + if (body.stream === true) { + const id = completion.id; + const created = completion.created; + sendSse(res, [ + { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }, + { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: { content }, finish_reason: null }], + }, + { + id, + object: "chat.completion.chunk", + created, + model, + choices: [{ index: 0, delta: {}, finish_reason: completion.choices[0].finish_reason }], + }, + ]); + return; + } + sendJson(res, 200, completion); +} + +const server = http.createServer((req, res) => { + const url = new URL(req.url || "/", `http://${req.headers.host || "localhost"}`); + Promise.resolve() + .then(async () => { + if (req.method === "GET" && (url.pathname === "/v1/models" || url.pathname === "/models")) { + await proxyModels(res); + return; + } + if (req.method === "POST" && (url.pathname === "/v1/chat/completions" || url.pathname === "/chat/completions")) { + await proxyChat(req, res); + return; + } + if (req.method === "GET" && url.pathname === "/health") { + sendJson(res, 200, { ok: true, upstream }); + return; + } + sendJson(res, 404, { error: { message: "not found", type: "not_found" } }); + }) + .catch((error) => { + sendJson(res, 500, { error: { message: error?.message || String(error), type: "proxy_error" } }); + }); +}); + +server.listen(listenPort, listenHost, () => { + console.log(`Ollama thinkless OpenAI proxy listening on http://${listenHost}:${listenPort}/v1 -> ${upstream}`); +});