Autopoll Ollama models for inference endpoints

This commit is contained in:
Codex
2026-04-24 17:55:37 +00:00
parent ecaefcf7c8
commit ee971a3035
3 changed files with 174 additions and 3 deletions
+86
View File
@@ -0,0 +1,86 @@
import { NextResponse } from "next/server"
const DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434"
const OLLAMA_BASE_URL = (process.env.OLLAMA_BASE_URL || DEFAULT_OLLAMA_URL).replace(/\/+$/, "")
type OllamaTag = {
name?: string
model?: string
modified_at?: string
size?: number
digest?: string
details?: {
family?: string
parameter_size?: string
quantization_level?: string
}
}
function formatBytes(size: unknown) {
if (typeof size !== "number" || !Number.isFinite(size) || size <= 0) return null
const units = ["B", "KB", "MB", "GB", "TB"]
let value = size
let unit = 0
while (value >= 1024 && unit < units.length - 1) {
value /= 1024
unit += 1
}
return `${value >= 10 || unit === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unit]}`
}
export async function GET() {
const startedAt = Date.now()
const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), 2500)
try {
const response = await fetch(`${OLLAMA_BASE_URL}/api/tags`, {
cache: "no-store",
signal: controller.signal,
})
const data = await response.json()
if (!response.ok) {
throw new Error(data?.error || `Ollama returned HTTP ${response.status}`)
}
const models = Array.isArray(data?.models)
? data.models.map((item: OllamaTag) => ({
name: item.model || item.name || "",
model: item.model || item.name || "",
modifiedAt: item.modified_at || null,
size: typeof item.size === "number" ? item.size : null,
sizeLabel: formatBytes(item.size),
digest: item.digest || null,
family: item.details?.family || null,
parameterSize: item.details?.parameter_size || null,
quantization: item.details?.quantization_level || null,
})).filter((item: { name: string }) => item.name)
: []
return NextResponse.json({
ok: true,
available: true,
baseUrl: OLLAMA_BASE_URL,
models,
count: models.length,
elapsedMs: Date.now() - startedAt,
})
} catch (error) {
const message = error instanceof Error && error.name === "AbortError"
? "Ollama probe timed out"
: error instanceof Error
? error.message
: "Failed to fetch Ollama models"
return NextResponse.json({
ok: false,
available: false,
baseUrl: OLLAMA_BASE_URL,
models: [],
count: 0,
elapsedMs: Date.now() - startedAt,
error: message,
}, { status: 200 })
} finally {
clearTimeout(timeout)
}
}
+80 -1
View File
@@ -1,6 +1,6 @@
"use client" "use client"
import { useEffect, useMemo, useState } from "react" import { useCallback, useEffect, useMemo, useState } from "react"
type InferenceRoute = { type InferenceRoute = {
configured: boolean configured: boolean
@@ -24,6 +24,16 @@ type InferenceResponse = {
providers: ProviderSummary[] providers: ProviderSummary[]
} }
type OllamaModel = {
name: string
model: string
modifiedAt: string | null
sizeLabel: string | null
family: string | null
parameterSize: string | null
quantization: string | null
}
const providerTypeOptions = [ const providerTypeOptions = [
{ value: "openai", label: "OpenAI compatible" }, { value: "openai", label: "OpenAI compatible" },
{ value: "nvidia", label: "NVIDIA" }, { value: "nvidia", label: "NVIDIA" },
@@ -55,11 +65,15 @@ export default function InferenceEndpointPanel() {
const [route, setRoute] = useState<"gateway" | "system">("gateway") const [route, setRoute] = useState<"gateway" | "system">("gateway")
const [timeout, setTimeoutValue] = useState("0") const [timeout, setTimeoutValue] = useState("0")
const [noVerify, setNoVerify] = useState(true) const [noVerify, setNoVerify] = useState(true)
const [ollamaModels, setOllamaModels] = useState<OllamaModel[]>([])
const [ollamaLoading, setOllamaLoading] = useState(false)
const [ollamaMessage, setOllamaMessage] = useState("")
const activeProvider = useMemo( const activeProvider = useMemo(
() => providers.find((provider) => provider.name === gateway.provider) ?? null, () => providers.find((provider) => provider.name === gateway.provider) ?? null,
[providers, gateway.provider] [providers, gateway.provider]
) )
const isOllamaEndpoint = name.toLowerCase().includes("ollama") || baseUrl.includes("11434")
async function load() { async function load() {
try { try {
@@ -84,12 +98,45 @@ export default function InferenceEndpointPanel() {
load() load()
}, []) }, [])
const loadOllamaModels = useCallback(async ({ silent = false }: { silent?: boolean } = {}) => {
try {
setOllamaLoading(true)
if (!silent) setOllamaMessage("")
const response = await fetch("/api/ollama/models", { cache: "no-store" })
const data = await response.json()
if (!data.available) {
setOllamaModels([])
setOllamaMessage(data.error || "Ollama is not reachable")
return
}
const models = Array.isArray(data.models) ? data.models : []
setOllamaModels(models)
setOllamaMessage(models.length > 0 ? `${models.length} Ollama model${models.length === 1 ? "" : "s"} available.` : "Ollama is reachable but has no models installed.")
if (models[0]?.name) setModel((current) => current || models[0].name)
} catch (error) {
setOllamaModels([])
setOllamaMessage(error instanceof Error ? error.message : "Failed to fetch Ollama models")
} finally {
setOllamaLoading(false)
}
}, [])
useEffect(() => {
if (!isOllamaEndpoint) return
loadOllamaModels({ silent: true })
const interval = window.setInterval(() => loadOllamaModels({ silent: true }), 10000)
return () => window.clearInterval(interval)
}, [isOllamaEndpoint, loadOllamaModels])
function selectProvider(provider: ProviderSummary) { function selectProvider(provider: ProviderSummary) {
if (!provider.name) return if (!provider.name) return
setName(provider.name) setName(provider.name)
setType(provider.type || "openai") setType(provider.type || "openai")
setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY") setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY")
if (provider.name.toLowerCase().includes("ollama")) {
setBaseUrl("http://host.docker.internal:11434/v1")
setApiKey("") setApiKey("")
}
setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`) setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`)
} }
@@ -181,7 +228,13 @@ export default function InferenceEndpointPanel() {
</div> </div>
<div className="space-y-2 md:col-span-2"> <div className="space-y-2 md:col-span-2">
<FieldLabel>Model</FieldLabel> <FieldLabel>Model</FieldLabel>
{isOllamaEndpoint && ollamaModels.length > 0 ? (
<select value={model} onChange={(event) => setModel(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]">
{ollamaModels.map((item) => <option key={item.name} value={item.name}>{item.name}</option>)}
</select>
) : (
<input value={model} onChange={(event) => setModel(event.target.value)} placeholder="provider/model-name" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> <input value={model} onChange={(event) => setModel(event.target.value)} placeholder="provider/model-name" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
)}
</div> </div>
<div className="space-y-2 md:col-span-2"> <div className="space-y-2 md:col-span-2">
<FieldLabel>Endpoint URL</FieldLabel> <FieldLabel>Endpoint URL</FieldLabel>
@@ -213,6 +266,32 @@ export default function InferenceEndpointPanel() {
Skip provider verification while saving Skip provider verification while saving
</label> </label>
{isOllamaEndpoint && (
<div className="rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] p-4">
<div className="flex items-center justify-between gap-4">
<div>
<h3 className="text-xs uppercase tracking-wider text-[var(--foreground)]">Ollama Models</h3>
<p className="mt-1 text-xs text-[var(--foreground-dim)]">{ollamaMessage || "Polling local Ollama every 10 seconds."}</p>
</div>
<button type="button" onClick={() => loadOllamaModels()} disabled={ollamaLoading} className="px-3 py-2 rounded-sm bg-[var(--background)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:bg-[var(--background-panel)] disabled:opacity-50">
{ollamaLoading ? "Polling..." : "Poll"}
</button>
</div>
{ollamaModels.length > 0 && (
<div className="mt-3 grid grid-cols-1 md:grid-cols-2 gap-2">
{ollamaModels.map((item) => (
<button key={item.name} type="button" onClick={() => setModel(item.name)} className={`rounded-sm border p-3 text-left ${model === item.name ? "border-[var(--nvidia-green)] bg-[rgba(118,185,0,0.08)]" : "border-[var(--border-subtle)] bg-[var(--background)]"}`}>
<div className="text-xs font-mono text-[var(--foreground)]">{item.name}</div>
<div className="mt-1 text-[11px] text-[var(--foreground-dim)]">
{[item.parameterSize, item.quantization, item.sizeLabel].filter(Boolean).join(" · ") || item.family || "local model"}
</div>
</button>
))}
</div>
)}
</div>
)}
<div className="flex items-center gap-3"> <div className="flex items-center gap-3">
<button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50"> <button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
{saving ? "Saving..." : "Save Endpoint"} {saving ? "Saving..." : "Save Endpoint"}
+6
View File
@@ -7,6 +7,7 @@ const createRoutePath = path.join(root, 'app/api/sandbox/create/route.ts')
const deleteRoutePath = path.join(root, 'app/api/sandbox/delete/route.ts') const deleteRoutePath = path.join(root, 'app/api/sandbox/delete/route.ts')
const telemetryRoutePath = path.join(root, 'app/api/telemetry/real/route.ts') const telemetryRoutePath = path.join(root, 'app/api/telemetry/real/route.ts')
const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts') const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts')
const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts')
const hookPath = path.join(root, 'app/hooks/useSandboxInventory.ts') const hookPath = path.join(root, 'app/hooks/useSandboxInventory.ts')
const pagePath = path.join(root, 'app/page.tsx') const pagePath = path.join(root, 'app/page.tsx')
const configPanelPath = path.join(root, 'app/components/ConfigurationPanel.tsx') const configPanelPath = path.join(root, 'app/components/ConfigurationPanel.tsx')
@@ -16,6 +17,7 @@ const createRouteSource = await readFile(createRoutePath, 'utf8')
const deleteRouteSource = await readFile(deleteRoutePath, 'utf8') const deleteRouteSource = await readFile(deleteRoutePath, 'utf8')
const telemetryRouteSource = await readFile(telemetryRoutePath, 'utf8') const telemetryRouteSource = await readFile(telemetryRoutePath, 'utf8')
const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8') const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8')
const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8')
const hookSource = await readFile(hookPath, 'utf8') const hookSource = await readFile(hookPath, 'utf8')
const pageSource = await readFile(pagePath, 'utf8') const pageSource = await readFile(pagePath, 'utf8')
const configPanelSource = await readFile(configPanelPath, 'utf8') const configPanelSource = await readFile(configPanelPath, 'utf8')
@@ -53,6 +55,10 @@ assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference
assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route') assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route')
assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config') assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config')
assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration') assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration')
assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models')
assert.match(inferencePanelSource, /window\.setInterval\(\(\) => loadOllamaModels\(\{ silent: true \}\), 10000\)/, 'settings UI must autopoll Ollama while selected')
assert.match(ollamaModelsRouteSource, /\/api\/tags/, 'Ollama models route must query the Ollama tags API')
assert.match(ollamaModelsRouteSource, /OLLAMA_BASE_URL/, 'Ollama models route must support a configurable Ollama base URL')
assert.match(pageSource, /<InferenceEndpointPanel \/>/, 'settings view must render inference endpoint panel') assert.match(pageSource, /<InferenceEndpointPanel \/>/, 'settings view must render inference endpoint panel')
assert.match(pageSource, /fetch\('\/api\/sandbox\/delete'/, 'destroy workflow must call the real sandbox delete endpoint') assert.match(pageSource, /fetch\('\/api\/sandbox\/delete'/, 'destroy workflow must call the real sandbox delete endpoint')