"use client" import { useCallback, useEffect, useMemo, useState } from "react" type InferenceRoute = { configured: boolean provider: string | null model: string | null version: string | null timeout: string | null } type ProviderSummary = { id: string | null name: string | null type: string | null credentialKeys: string[] configKeys: string[] } type InferenceResponse = { gateway: InferenceRoute system: InferenceRoute providers: ProviderSummary[] } type OllamaModel = { name: string model: string modifiedAt: string | null sizeLabel: string | null family: string | null parameterSize: string | null quantization: string | null } const providerTypeOptions = [ { value: "openai", label: "OpenAI compatible" }, { value: "nvidia", label: "NVIDIA" }, { value: "anthropic", label: "Anthropic" }, { value: "generic", label: "Generic" }, ] function emptyRoute(): InferenceRoute { return { configured: false, provider: null, model: null, version: null, timeout: null } } function FieldLabel({ children }: { children: React.ReactNode }) { return } export default function InferenceEndpointPanel() { const [loading, setLoading] = useState(true) const [saving, setSaving] = useState(false) const [message, setMessage] = useState("") const [providers, setProviders] = useState([]) const [gateway, setGateway] = useState(emptyRoute) const [system, setSystem] = useState(emptyRoute) const [name, setName] = useState("nvidia-nim") const [type, setType] = useState("openai") const [model, setModel] = useState("nvidia/nemotron-3-super-120b-a12b") const [baseUrl, setBaseUrl] = useState("https://integrate.api.nvidia.com/v1") const [credentialKey, setCredentialKey] = useState("NVIDIA_API_KEY") const [apiKey, setApiKey] = useState("") const [route, setRoute] = useState<"gateway" | "system">("gateway") const [timeout, setTimeoutValue] = useState("0") const [noVerify, setNoVerify] = useState(true) const [ollamaModels, setOllamaModels] = useState([]) const [ollamaLoading, setOllamaLoading] = useState(false) const [ollamaMessage, setOllamaMessage] = useState("") const activeProvider = useMemo( () => providers.find((provider) => provider.name === gateway.provider) ?? null, [providers, gateway.provider] ) const isOllamaEndpoint = name.toLowerCase().includes("ollama") || baseUrl.includes("11434") async function load() { try { setLoading(true) setMessage("") const response = await fetch("/api/inference", { cache: "no-store" }) const data = await response.json() as InferenceResponse & { error?: string } if (!response.ok) throw new Error(data.error || "Failed to load inference configuration") setGateway(data.gateway ?? emptyRoute()) setSystem(data.system ?? emptyRoute()) setProviders(Array.isArray(data.providers) ? data.providers : []) if (data.gateway?.provider) setName(data.gateway.provider) if (data.gateway?.model) setModel(data.gateway.model) } catch (error) { setMessage(error instanceof Error ? error.message : "Failed to load inference configuration") } finally { setLoading(false) } } useEffect(() => { load() }, []) const loadOllamaModels = useCallback(async ({ silent = false }: { silent?: boolean } = {}) => { try { setOllamaLoading(true) if (!silent) setOllamaMessage("") const response = await fetch("/api/ollama/models", { cache: "no-store" }) const data = await response.json() if (!data.available) { setOllamaModels([]) setOllamaMessage(data.error || "Ollama is not reachable") return } const models = Array.isArray(data.models) ? data.models : [] setOllamaModels(models) setOllamaMessage(models.length > 0 ? `${models.length} Ollama model${models.length === 1 ? "" : "s"} available.` : "Ollama is reachable but has no models installed.") if (models[0]?.name) setModel((current) => current || models[0].name) } catch (error) { setOllamaModels([]) setOllamaMessage(error instanceof Error ? error.message : "Failed to fetch Ollama models") } finally { setOllamaLoading(false) } }, []) useEffect(() => { if (!isOllamaEndpoint) return loadOllamaModels({ silent: true }) const interval = window.setInterval(() => loadOllamaModels({ silent: true }), 10000) return () => window.clearInterval(interval) }, [isOllamaEndpoint, loadOllamaModels]) function selectProvider(provider: ProviderSummary) { if (!provider.name) return setName(provider.name) setType(provider.type || "openai") setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY") if (provider.name.toLowerCase().includes("ollama")) { setBaseUrl("http://host.docker.internal:11434/v1") setApiKey("") } setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`) } async function saveEndpoint() { try { setSaving(true) setMessage("") const response = await fetch("/api/inference", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ name, type, model, baseUrl, credentialKey, apiKey, route, timeout: Number(timeout || "0"), noVerify, makeActive: true, }), }) const data = await response.json() if (!response.ok) throw new Error(data.error || "Failed to save inference endpoint") setGateway(data.gateway ?? gateway) setSystem(data.system ?? system) setApiKey("") setMessage(`Inference endpoint '${name}' saved and routed to ${route === "system" ? "system" : "sandbox"} inference.`) await load() } catch (error) { setMessage(error instanceof Error ? error.message : "Failed to save inference endpoint") } finally { setSaving(false) } } return (

INFERENCE ENDPOINTS

Configure the OpenShell gateway route used by sandboxes through inference.local.

{loading ? (
Loading inference routes...
) : (

Sandbox Route

{gateway.configured ? gateway.provider : "not configured"}

{gateway.model || "No model selected"}

{gateway.timeout &&

Timeout {gateway.timeout}

}

System Route

{system.configured ? system.provider : "not configured"}

{system.model || "No model selected"}

{system.timeout &&

Timeout {system.timeout}

}
Provider Name setName(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
Provider Type
Model {isOllamaEndpoint && ollamaModels.length > 0 ? ( ) : ( setModel(event.target.value)} placeholder="provider/model-name" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> )}
Endpoint URL setBaseUrl(event.target.value)} placeholder="https://example.com/v1" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
Credential Env Key setCredentialKey(event.target.value.toUpperCase())} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
API Key setApiKey(event.target.value)} type="password" placeholder={activeProvider ? "leave blank to keep stored key" : "stored in OpenShell provider"} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
Route
Timeout Seconds setTimeoutValue(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
{isOllamaEndpoint && (

Ollama Models

{ollamaMessage || "Polling local Ollama every 10 seconds."}

{ollamaModels.length > 0 && (
{ollamaModels.map((item) => ( ))}
)}
)}
{message &&

{message}

}

Configured Providers

{providers.length === 0 ? (
No providers configured.
) : (
{providers.map((provider) => ( ))}
)}
)}
) }