"use client" import { useCallback, useEffect, useMemo, useState } from "react" import type { SandboxInventoryItem } from "../hooks/useSandboxInventory" type ProviderSummary = { id: string | null name: string | null type: string | null } type InferenceRouteStatus = { configured: boolean provider: string | null model: string | null version: string | null timeout: string | null } type OllamaModel = { name: string sizeLabel: string | null parameterSize: string | null quantization: string | null } type SandboxInferenceRoute = { id: string provider: string model: string enabled: boolean label: string } type SandboxInferenceConfig = { sandboxId: string provider: string primaryModel: string models: string[] routes: SandboxInferenceRoute[] primaryRouteId: string updatedAt: string | null } function makeRoute(provider: string, model: string, label = ""): SandboxInferenceRoute { return { id: `${provider}::${model}`, provider, model, enabled: true, label, } } function routeKey(route: Pick) { return `${route.provider}::${route.model}` } function dedupeRoutes(routes: SandboxInferenceRoute[]) { return Array.from(new Map(routes .filter((route) => route.provider.trim() && route.model.trim()) .map((route) => [route.id || routeKey(route), { ...route, id: route.id || routeKey(route) }])).values()) } export default function SandboxInferencePanel({ sandbox, embedded = false, showHeader = true, }: { sandbox: SandboxInventoryItem embedded?: boolean showHeader?: boolean }) { const [loading, setLoading] = useState(true) const [saving, setSaving] = useState(false) const [applying, setApplying] = useState(false) const [message, setMessage] = useState("") const [providers, setProviders] = useState([]) const [verifiedRoutes, setVerifiedRoutes] = useState>([]) const [routes, setRoutes] = useState([]) const [primaryRouteId, setPrimaryRouteId] = useState("") const [draftProvider, setDraftProvider] = useState("") const [draftModel, setDraftModel] = useState("") const [draftLabel, setDraftLabel] = useState("") const [updatedAt, setUpdatedAt] = useState(null) const [ollamaModels, setOllamaModels] = useState([]) const [ollamaLoading, setOllamaLoading] = useState(false) const draftProviderIsOllama = draftProvider.toLowerCase().includes("ollama") const anyOllamaRoute = routes.some((route) => route.provider.toLowerCase().includes("ollama")) const shouldPollOllama = draftProviderIsOllama || anyOllamaRoute const primaryRoute = useMemo( () => routes.find((route) => route.id === primaryRouteId) || routes[0] || null, [routes, primaryRouteId] ) const loadOllamaModels = useCallback(async () => { try { setOllamaLoading(true) const response = await fetch("/api/ollama/models", { cache: "no-store" }) const data = await response.json() setOllamaModels(data.available && Array.isArray(data.models) ? data.models : []) } finally { setOllamaLoading(false) } }, []) const load = useCallback(async () => { try { setLoading(true) setMessage("") const [inferenceResponse, configResponse] = await Promise.all([ fetch("/api/inference", { cache: "no-store" }), fetch(`/api/sandbox/${encodeURIComponent(sandbox.id)}/inference`, { cache: "no-store" }), ]) const inferenceData = await inferenceResponse.json() const configData = await configResponse.json() if (!inferenceResponse.ok) throw new Error(inferenceData.error || "Failed to load providers") if (!configResponse.ok) throw new Error(configData.error || "Failed to load sandbox inference config") const nextProviders = Array.isArray(inferenceData.providers) ? inferenceData.providers : [] const config = configData.config as SandboxInferenceConfig const gatewayRoute = inferenceData.gateway as InferenceRouteStatus | undefined const systemRoute = inferenceData.system as InferenceRouteStatus | undefined const nextVerifiedRoutes = [ gatewayRoute?.configured && gatewayRoute.provider && gatewayRoute.model ? { ...makeRoute(gatewayRoute.provider, gatewayRoute.model, "Active sandbox route"), scope: "Sandbox" as const } : null, systemRoute?.configured && systemRoute.provider && systemRoute.model ? { ...makeRoute(systemRoute.provider, systemRoute.model, "Active system route"), scope: "System" as const } : null, ].filter((route): route is SandboxInferenceRoute & { scope: "Sandbox" | "System" } => Boolean(route)) const fallbackProvider = gatewayRoute?.provider || nextProviders[0]?.name || "" const fallbackModel = gatewayRoute?.model || "" const nextRoutes = dedupeRoutes(Array.isArray(config?.routes) && config.routes.length > 0 ? config.routes : fallbackProvider && fallbackModel ? [makeRoute(fallbackProvider, fallbackModel, "Gateway default")] : []) setProviders(nextProviders) setVerifiedRoutes(nextVerifiedRoutes) setRoutes(nextRoutes) setPrimaryRouteId(config?.primaryRouteId || nextRoutes[0]?.id || "") setDraftProvider(fallbackProvider) setDraftModel("") setDraftLabel("") setUpdatedAt(config?.updatedAt || null) } catch (error) { setMessage(error instanceof Error ? error.message : "Failed to load sandbox inference config") } finally { setLoading(false) } }, [sandbox.id]) useEffect(() => { load() }, [load]) useEffect(() => { if (!shouldPollOllama) return loadOllamaModels() const interval = window.setInterval(loadOllamaModels, 10000) return () => window.clearInterval(interval) }, [shouldPollOllama, loadOllamaModels]) function addRoute(provider = draftProvider, model = draftModel, label = draftLabel) { const cleanProvider = provider.trim() const cleanModel = model.trim() if (!cleanProvider || !cleanModel) { setMessage("Choose a provider and model before adding a route.") return } const nextRoute = makeRoute(cleanProvider, cleanModel, label.trim()) setRoutes((current) => { const next = dedupeRoutes([...current, nextRoute]) if (!primaryRouteId) setPrimaryRouteId(nextRoute.id) return next }) setDraftModel("") setDraftLabel("") setMessage("") } function updateRoute(id: string, updates: Partial) { setRoutes((current) => current.map((route) => route.id === id ? { ...route, ...updates } : route)) } function removeRoute(id: string) { setRoutes((current) => { const next = current.filter((route) => route.id !== id) if (primaryRouteId === id) setPrimaryRouteId(next[0]?.id || "") return next }) } async function save() { try { setSaving(true) setMessage("") const cleanRoutes = dedupeRoutes(routes) if (cleanRoutes.length === 0) throw new Error("Add at least one provider/model route.") const primary = cleanRoutes.find((route) => route.id === primaryRouteId) || cleanRoutes[0] const response = await fetch(`/api/sandbox/${encodeURIComponent(sandbox.id)}/inference`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ provider: primary.provider, primaryModel: primary.model, models: cleanRoutes.map((route) => route.model), routes: cleanRoutes, primaryRouteId: primary.id, }), }) const data = await response.json() if (!response.ok) throw new Error(data.error || "Failed to save sandbox inference config") const config = data.config as SandboxInferenceConfig setRoutes(config.routes) setPrimaryRouteId(config.primaryRouteId) setUpdatedAt(config.updatedAt) setMessage(`Saved ${config.routes.length} inference route${config.routes.length === 1 ? "" : "s"} for ${sandbox.name}.`) } catch (error) { setMessage(error instanceof Error ? error.message : "Failed to save sandbox inference config") } finally { setSaving(false) } } async function applyToContainer() { try { setApplying(true) setMessage("") const cleanRoutes = dedupeRoutes(routes) if (cleanRoutes.length === 0) throw new Error("Add at least one provider/model route before applying.") await save() const response = await fetch(`/api/sandbox/${encodeURIComponent(sandbox.id)}/inference/apply`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sandboxName: sandbox.name }), }) const data = await response.json() if (!response.ok) throw new Error(data.note ? `${data.error}\n\n${data.note}` : data.error || "Failed to apply routes") setMessage(`${data.routesApplied} route${data.routesApplied === 1 ? "" : "s"} applied to ${sandbox.name}. ${data.note}`) } catch (error) { setMessage(error instanceof Error ? error.message : "Failed to apply routes") } finally { setApplying(false) } } return (
{showHeader && (

{sandbox.name} - INFERENCE ROUTES

Enable multiple endpoint/model routes for this sandbox.

)} {loading ? (
Loading inference routes...
) : (
{!showHeader && (
)} {primaryRoute && (

Primary Route

{primaryRoute.provider}

{primaryRoute.model}

)} {verifiedRoutes.length > 0 && (
Verified Working Routes
OpenShell active
{verifiedRoutes.map((route) => ( ))}
)}
setDraftModel(event.target.value)} placeholder="model id" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
setDraftLabel(event.target.value)} placeholder="optional" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
{draftProviderIsOllama && (
Ollama Models
{ollamaModels.length === 0 ? (

No local Ollama models reported.

) : (
{ollamaModels.map((item) => ( ))}
)}
)}
Enabled Routes
{routes.length === 0 ? (
No routes enabled for this sandbox.
) : (
{routes.map((route) => (
{route.provider} updateRoute(route.id, { model: event.target.value })} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> updateRoute(route.id, { label: event.target.value })} placeholder="label" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
))}
)}
{updatedAt && Updated {new Date(updatedAt).toLocaleString()}} {message && {message}}
)}
) }