diff --git a/app/api/sandbox/[sandboxId]/inference/route.ts b/app/api/sandbox/[sandboxId]/inference/route.ts index 985d009..ccf7894 100644 --- a/app/api/sandbox/[sandboxId]/inference/route.ts +++ b/app/api/sandbox/[sandboxId]/inference/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from "next/server" import { getSandboxInferenceConfig, saveSandboxInferenceConfig, + type SandboxInferenceRoute, } from "@/app/lib/sandboxInferenceStore" function validateSandboxId(value: string) { @@ -19,6 +20,24 @@ function parseModels(value: unknown) { .filter(Boolean) } +function parseRoutes(value: unknown): SandboxInferenceRoute[] { + if (!Array.isArray(value)) return [] + return value.map((item) => { + const provider = typeof item?.provider === "string" ? item.provider.trim() : "" + const model = typeof item?.model === "string" ? item.model.trim() : "" + const id = typeof item?.id === "string" && item.id.trim() + ? item.id.trim() + : `${provider}::${model}` + return { + id, + provider, + model, + enabled: item?.enabled !== false, + label: typeof item?.label === "string" ? item.label.trim() : "", + } + }).filter((item) => item.provider && item.model) +} + export async function GET( _request: Request, { params }: { params: Promise<{ sandboxId: string }> } @@ -43,14 +62,18 @@ export async function POST( const provider = typeof body?.provider === "string" ? body.provider.trim() : "" const primaryModel = typeof body?.primaryModel === "string" ? body.primaryModel.trim() : "" const models = parseModels(body?.models) + const routes = parseRoutes(body?.routes) + const primaryRouteId = typeof body?.primaryRouteId === "string" ? body.primaryRouteId.trim() : "" - if (!provider) throw new Error("provider is required") - if (!primaryModel && models.length === 0) throw new Error("at least one model is required") + if (routes.length === 0 && !provider) throw new Error("provider is required") + if (routes.length === 0 && !primaryModel && models.length === 0) throw new Error("at least one model is required") const config = await saveSandboxInferenceConfig(validateSandboxId(sandboxId), { provider, primaryModel: primaryModel || models[0], models, + routes, + primaryRouteId, }) return NextResponse.json({ ok: true, config }) } catch (error) { diff --git a/app/components/SandboxInferencePanel.tsx b/app/components/SandboxInferencePanel.tsx index b9641d2..295f5a6 100644 --- a/app/components/SandboxInferencePanel.tsx +++ b/app/components/SandboxInferencePanel.tsx @@ -16,20 +16,42 @@ type OllamaModel = { quantization: string | null } +type SandboxInferenceRoute = { + id: string + provider: string + model: string + enabled: boolean + label: string +} + type SandboxInferenceConfig = { sandboxId: string provider: string primaryModel: string models: string[] + routes: SandboxInferenceRoute[] + primaryRouteId: string updatedAt: string | null } -function emptyConfig(sandboxId: string): SandboxInferenceConfig { - return { sandboxId, provider: "", primaryModel: "", models: [], updatedAt: null } +function makeRoute(provider: string, model: string, label = ""): SandboxInferenceRoute { + return { + id: `${provider}::${model}`, + provider, + model, + enabled: true, + label, + } } -function uniqueModels(values: string[]) { - return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean))) +function routeKey(route: Pick) { + return `${route.provider}::${route.model}` +} + +function dedupeRoutes(routes: SandboxInferenceRoute[]) { + return Array.from(new Map(routes + .filter((route) => route.provider.trim() && route.model.trim()) + .map((route) => [route.id || routeKey(route), { ...route, id: route.id || routeKey(route) }])).values()) } export default function SandboxInferencePanel({ sandbox }: { sandbox: SandboxInventoryItem }) { @@ -37,17 +59,21 @@ export default function SandboxInferencePanel({ sandbox }: { sandbox: SandboxInv const [saving, setSaving] = useState(false) const [message, setMessage] = useState("") const [providers, setProviders] = useState([]) - const [provider, setProvider] = useState("") - const [primaryModel, setPrimaryModel] = useState("") - const [modelsText, setModelsText] = useState("") + const [routes, setRoutes] = useState([]) + const [primaryRouteId, setPrimaryRouteId] = useState("") + const [draftProvider, setDraftProvider] = useState("") + const [draftModel, setDraftModel] = useState("") + const [draftLabel, setDraftLabel] = useState("") const [updatedAt, setUpdatedAt] = useState(null) const [ollamaModels, setOllamaModels] = useState([]) const [ollamaLoading, setOllamaLoading] = useState(false) - const isOllamaProvider = provider.toLowerCase().includes("ollama") - const selectedModels = useMemo( - () => uniqueModels([primaryModel, ...modelsText.split(/\r?\n/)]), - [primaryModel, modelsText] + const draftProviderIsOllama = draftProvider.toLowerCase().includes("ollama") + const anyOllamaRoute = routes.some((route) => route.provider.toLowerCase().includes("ollama")) + const shouldPollOllama = draftProviderIsOllama || anyOllamaRoute + const primaryRoute = useMemo( + () => routes.find((route) => route.id === primaryRouteId) || routes[0] || null, + [routes, primaryRouteId] ) const loadOllamaModels = useCallback(async () => { @@ -75,16 +101,22 @@ export default function SandboxInferencePanel({ sandbox }: { sandbox: SandboxInv if (!configResponse.ok) throw new Error(configData.error || "Failed to load sandbox inference config") const nextProviders = Array.isArray(inferenceData.providers) ? inferenceData.providers : [] - const config = (configData.config ?? emptyConfig(sandbox.id)) as SandboxInferenceConfig + const config = configData.config as SandboxInferenceConfig const fallbackProvider = inferenceData.gateway?.provider || nextProviders[0]?.name || "" const fallbackModel = inferenceData.gateway?.model || "" - const models = config.models.length > 0 ? config.models : (fallbackModel ? [fallbackModel] : []) + const nextRoutes = dedupeRoutes(Array.isArray(config?.routes) && config.routes.length > 0 + ? config.routes + : fallbackProvider && fallbackModel + ? [makeRoute(fallbackProvider, fallbackModel, "Gateway default")] + : []) setProviders(nextProviders) - setProvider(config.provider || fallbackProvider) - setPrimaryModel(config.primaryModel || models[0] || "") - setModelsText(models.join("\n")) - setUpdatedAt(config.updatedAt) + setRoutes(nextRoutes) + setPrimaryRouteId(config?.primaryRouteId || nextRoutes[0]?.id || "") + setDraftProvider(fallbackProvider) + setDraftModel("") + setDraftLabel("") + setUpdatedAt(config?.updatedAt || null) } catch (error) { setMessage(error instanceof Error ? error.message : "Failed to load sandbox inference config") } finally { @@ -97,42 +129,67 @@ export default function SandboxInferencePanel({ sandbox }: { sandbox: SandboxInv }, [load]) useEffect(() => { - if (!isOllamaProvider) return + if (!shouldPollOllama) return loadOllamaModels() const interval = window.setInterval(loadOllamaModels, 10000) return () => window.clearInterval(interval) - }, [isOllamaProvider, loadOllamaModels]) + }, [shouldPollOllama, loadOllamaModels]) - function toggleModel(modelName: string) { - const next = selectedModels.includes(modelName) - ? selectedModels.filter((item) => item !== modelName) - : [...selectedModels, modelName] - setModelsText(next.join("\n")) - setPrimaryModel((current) => next.includes(current) ? current : next[0] || "") + function addRoute(provider = draftProvider, model = draftModel, label = draftLabel) { + const cleanProvider = provider.trim() + const cleanModel = model.trim() + if (!cleanProvider || !cleanModel) { + setMessage("Choose a provider and model before adding a route.") + return + } + const nextRoute = makeRoute(cleanProvider, cleanModel, label.trim()) + setRoutes((current) => { + const next = dedupeRoutes([...current, nextRoute]) + if (!primaryRouteId) setPrimaryRouteId(nextRoute.id) + return next + }) + setDraftModel("") + setDraftLabel("") + setMessage("") + } + + function updateRoute(id: string, updates: Partial) { + setRoutes((current) => current.map((route) => route.id === id ? { ...route, ...updates } : route)) + } + + function removeRoute(id: string) { + setRoutes((current) => { + const next = current.filter((route) => route.id !== id) + if (primaryRouteId === id) setPrimaryRouteId(next[0]?.id || "") + return next + }) } async function save() { try { setSaving(true) setMessage("") - const models = selectedModels + const cleanRoutes = dedupeRoutes(routes) + if (cleanRoutes.length === 0) throw new Error("Add at least one provider/model route.") + const primary = cleanRoutes.find((route) => route.id === primaryRouteId) || cleanRoutes[0] const response = await fetch(`/api/sandbox/${encodeURIComponent(sandbox.id)}/inference`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - provider, - primaryModel: primaryModel || models[0], - models, + provider: primary.provider, + primaryModel: primary.model, + models: cleanRoutes.map((route) => route.model), + routes: cleanRoutes, + primaryRouteId: primary.id, }), }) const data = await response.json() if (!response.ok) throw new Error(data.error || "Failed to save sandbox inference config") const config = data.config as SandboxInferenceConfig - setProvider(config.provider) - setPrimaryModel(config.primaryModel) - setModelsText(config.models.join("\n")) + setRoutes(config.routes) + setPrimaryRouteId(config.primaryRouteId) setUpdatedAt(config.updatedAt) - setMessage(`Saved inference profile for ${sandbox.name}.`) + setMessage(`Saved ${config.routes.length} inference route${config.routes.length === 1 ? "" : "s"} for ${sandbox.name}.`) } catch (error) { setMessage(error instanceof Error ? error.message : "Failed to save sandbox inference config") } finally { @@ -145,10 +202,10 @@ export default function SandboxInferencePanel({ sandbox }: { sandbox: SandboxInv

- {sandbox.name} - INFERENCE PROFILE + {sandbox.name} - INFERENCE ROUTES

- Container-specific provider and model selection. + Enable multiple endpoint/model routes for this sandbox.

- {isOllamaProvider && ( + {draftProviderIsOllama && (
Ollama Models
@@ -191,7 +263,7 @@ export default function SandboxInferencePanel({ sandbox }: { sandbox: SandboxInv ) : (
{ollamaModels.map((item) => ( -