From ee971a303516ef11669f13134419182d893372d2 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 24 Apr 2026 17:55:37 +0000 Subject: [PATCH] Autopoll Ollama models for inference endpoints --- app/api/ollama/models/route.ts | 86 +++++++++++++++++++++++ app/components/InferenceEndpointPanel.tsx | 85 +++++++++++++++++++++- tests/sandbox-lifecycle-check.mjs | 6 ++ 3 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 app/api/ollama/models/route.ts diff --git a/app/api/ollama/models/route.ts b/app/api/ollama/models/route.ts new file mode 100644 index 0000000..d68266d --- /dev/null +++ b/app/api/ollama/models/route.ts @@ -0,0 +1,86 @@ +import { NextResponse } from "next/server" + +const DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434" +const OLLAMA_BASE_URL = (process.env.OLLAMA_BASE_URL || DEFAULT_OLLAMA_URL).replace(/\/+$/, "") + +type OllamaTag = { + name?: string + model?: string + modified_at?: string + size?: number + digest?: string + details?: { + family?: string + parameter_size?: string + quantization_level?: string + } +} + +function formatBytes(size: unknown) { + if (typeof size !== "number" || !Number.isFinite(size) || size <= 0) return null + const units = ["B", "KB", "MB", "GB", "TB"] + let value = size + let unit = 0 + while (value >= 1024 && unit < units.length - 1) { + value /= 1024 + unit += 1 + } + return `${value >= 10 || unit === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unit]}` +} + +export async function GET() { + const startedAt = Date.now() + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 2500) + + try { + const response = await fetch(`${OLLAMA_BASE_URL}/api/tags`, { + cache: "no-store", + signal: controller.signal, + }) + const data = await response.json() + if (!response.ok) { + throw new Error(data?.error || `Ollama returned HTTP ${response.status}`) + } + + const models = Array.isArray(data?.models) + ? data.models.map((item: OllamaTag) => ({ + name: item.model || item.name || "", + model: item.model || item.name || "", + modifiedAt: item.modified_at || null, + size: typeof item.size === "number" ? item.size : null, + sizeLabel: formatBytes(item.size), + digest: item.digest || null, + family: item.details?.family || null, + parameterSize: item.details?.parameter_size || null, + quantization: item.details?.quantization_level || null, + })).filter((item: { name: string }) => item.name) + : [] + + return NextResponse.json({ + ok: true, + available: true, + baseUrl: OLLAMA_BASE_URL, + models, + count: models.length, + elapsedMs: Date.now() - startedAt, + }) + } catch (error) { + const message = error instanceof Error && error.name === "AbortError" + ? "Ollama probe timed out" + : error instanceof Error + ? error.message + : "Failed to fetch Ollama models" + return NextResponse.json({ + ok: false, + available: false, + baseUrl: OLLAMA_BASE_URL, + models: [], + count: 0, + elapsedMs: Date.now() - startedAt, + error: message, + }, { status: 200 }) + } finally { + clearTimeout(timeout) + } +} diff --git a/app/components/InferenceEndpointPanel.tsx b/app/components/InferenceEndpointPanel.tsx index 649350d..c4ebf06 100644 --- a/app/components/InferenceEndpointPanel.tsx +++ b/app/components/InferenceEndpointPanel.tsx @@ -1,6 +1,6 @@ "use client" -import { useEffect, useMemo, useState } from "react" +import { useCallback, useEffect, useMemo, useState } from "react" type InferenceRoute = { configured: boolean @@ -24,6 +24,16 @@ type InferenceResponse = { providers: ProviderSummary[] } +type OllamaModel = { + name: string + model: string + modifiedAt: string | null + sizeLabel: string | null + family: string | null + parameterSize: string | null + quantization: string | null +} + const providerTypeOptions = [ { value: "openai", label: "OpenAI compatible" }, { value: "nvidia", label: "NVIDIA" }, @@ -55,11 +65,15 @@ export default function InferenceEndpointPanel() { const [route, setRoute] = useState<"gateway" | "system">("gateway") const [timeout, setTimeoutValue] = useState("0") const [noVerify, setNoVerify] = useState(true) + const [ollamaModels, setOllamaModels] = useState([]) + const [ollamaLoading, setOllamaLoading] = useState(false) + const [ollamaMessage, setOllamaMessage] = useState("") const activeProvider = useMemo( () => providers.find((provider) => provider.name === gateway.provider) ?? null, [providers, gateway.provider] ) + const isOllamaEndpoint = name.toLowerCase().includes("ollama") || baseUrl.includes("11434") async function load() { try { @@ -84,12 +98,45 @@ export default function InferenceEndpointPanel() { load() }, []) + const loadOllamaModels = useCallback(async ({ silent = false }: { silent?: boolean } = {}) => { + try { + setOllamaLoading(true) + if (!silent) setOllamaMessage("") + const response = await fetch("/api/ollama/models", { cache: "no-store" }) + const data = await response.json() + if (!data.available) { + setOllamaModels([]) + setOllamaMessage(data.error || "Ollama is not reachable") + return + } + const models = Array.isArray(data.models) ? data.models : [] + setOllamaModels(models) + setOllamaMessage(models.length > 0 ? `${models.length} Ollama model${models.length === 1 ? "" : "s"} available.` : "Ollama is reachable but has no models installed.") + if (models[0]?.name) setModel((current) => current || models[0].name) + } catch (error) { + setOllamaModels([]) + setOllamaMessage(error instanceof Error ? error.message : "Failed to fetch Ollama models") + } finally { + setOllamaLoading(false) + } + }, []) + + useEffect(() => { + if (!isOllamaEndpoint) return + loadOllamaModels({ silent: true }) + const interval = window.setInterval(() => loadOllamaModels({ silent: true }), 10000) + return () => window.clearInterval(interval) + }, [isOllamaEndpoint, loadOllamaModels]) + function selectProvider(provider: ProviderSummary) { if (!provider.name) return setName(provider.name) setType(provider.type || "openai") setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY") - setApiKey("") + if (provider.name.toLowerCase().includes("ollama")) { + setBaseUrl("http://host.docker.internal:11434/v1") + setApiKey("") + } setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`) } @@ -181,7 +228,13 @@ export default function InferenceEndpointPanel() {
Model - setModel(event.target.value)} placeholder="provider/model-name" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> + {isOllamaEndpoint && ollamaModels.length > 0 ? ( + + ) : ( + setModel(event.target.value)} placeholder="provider/model-name" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> + )}
Endpoint URL @@ -213,6 +266,32 @@ export default function InferenceEndpointPanel() { Skip provider verification while saving + {isOllamaEndpoint && ( +
+
+
+

Ollama Models

+

{ollamaMessage || "Polling local Ollama every 10 seconds."}

+
+ +
+ {ollamaModels.length > 0 && ( +
+ {ollamaModels.map((item) => ( + + ))} +
+ )} +
+ )} +