"use client" import { useCallback, useEffect, useMemo, useState } from "react" type InferenceRoute = { configured: boolean provider: string | null model: string | null version: string | null timeout: string | null } type ProviderSummary = { id: string | null name: string | null type: string | null credentialKeys: string[] configKeys: string[] } type InferenceResponse = { gateway: InferenceRoute system: InferenceRoute providers: ProviderSummary[] } type OllamaModel = { name: string model: string modifiedAt: string | null sizeLabel: string | null family: string | null parameterSize: string | null quantization: string | null } const providerTypeOptions = [ { value: "openai", label: "OpenAI compatible" }, { value: "vllm", label: "vLLM" }, { value: "anthropic", label: "Anthropic" }, { value: "generic", label: "Generic" }, ] const OLLAMA_PROVIDER_NAME = "ollama-local" const OLLAMA_BASE_URL = "http://host.docker.internal:11434/v1" const DEFAULT_VLLM_CONTAINER = "vllm-bigboy" const DEFAULT_VLLM_IMAGE = "nvcr.io/nvidia/vllm:26.03.post1-py3" const DEFAULT_VLLM_MODEL = "vllm-local" const DEFAULT_VLLM_MAX_MODEL_LEN = "32768" const DEFAULT_VLLM_YARN_FACTOR = "1.0" const DEFAULT_VLLM_ORIGINAL_CONTEXT = "32768" function isVllmEndpoint(name: string, baseUrl: string) { return [name, baseUrl].some((value) => /vllm/i.test(value)) } function shellQuote(value: string) { const trimmed = value.trim() if (!trimmed) return "''" return /^[A-Za-z0-9_./:@%+=,-]+$/.test(trimmed) ? trimmed : `'${trimmed.replace(/'/g, "'\\''")}'` } function commandLine(parts: string[]) { return parts.filter(Boolean).join(" \\\n ") } function emptyRoute(): InferenceRoute { return { configured: false, provider: null, model: null, version: null, timeout: null } } function FieldLabel({ children }: { children: React.ReactNode }) { return } function VllmAdvancedConfiguration({ model, port, }: { model: string port: string }) { const [containerName, setContainerName] = useState(DEFAULT_VLLM_CONTAINER) const [image, setImage] = useState(DEFAULT_VLLM_IMAGE) const [gpuDevice, setGpuDevice] = useState("0") const [hostPort, setHostPort] = useState("8000") const [containerPort, setContainerPort] = useState("8000") const [tensorParallelSize, setTensorParallelSize] = useState("1") const [maxModelLen, setMaxModelLen] = useState(DEFAULT_VLLM_MAX_MODEL_LEN) const [gpuMemoryUtilization, setGpuMemoryUtilization] = useState("0.85") const [dtype, setDtype] = useState("bfloat16") const [kvCacheDtype, setKvCacheDtype] = useState("fp8") const [maxNumBatchedTokens, setMaxNumBatchedTokens] = useState("8192") const [maxNumSeqs, setMaxNumSeqs] = useState("") const [toolCallParser, setToolCallParser] = useState("hermes") const [allowLongMaxModelLen, setAllowLongMaxModelLen] = useState(false) const [enableYarnScaling, setEnableYarnScaling] = useState(false) const [yarnFactor, setYarnFactor] = useState(DEFAULT_VLLM_YARN_FACTOR) const [originalContext, setOriginalContext] = useState(DEFAULT_VLLM_ORIGINAL_CONTEXT) const [enableChunkedPrefill, setEnableChunkedPrefill] = useState(true) const [enableAutoToolChoice, setEnableAutoToolChoice] = useState(true) const [calculateKvScales, setCalculateKvScales] = useState(false) const [copied, setCopied] = useState(false) const [applying, setApplying] = useState(false) const [restarting, setRestarting] = useState(false) const [restartMessage, setRestartMessage] = useState("") useEffect(() => { if (port && hostPort === "8000") setHostPort(port) if (port && containerPort === "8000") setContainerPort(port) }, [containerPort, hostPort, port]) const maxInputWithDefaultOutput = useMemo(() => { const parsed = Number.parseInt(maxModelLen, 10) return Number.isFinite(parsed) ? Math.max(0, parsed - 4096) : 0 }, [maxModelLen]) const hfOverridesJson = useMemo(() => { const factor = Number.parseFloat(yarnFactor) const originalMaxPositionEmbeddings = Number.parseInt(originalContext, 10) const maxPositionEmbeddings = Number.parseInt(maxModelLen, 10) return JSON.stringify({ max_position_embeddings: Number.isFinite(maxPositionEmbeddings) && maxPositionEmbeddings > 0 ? maxPositionEmbeddings : Number.parseInt(DEFAULT_VLLM_MAX_MODEL_LEN, 10), rope_scaling: { type: "yarn", factor: Number.isFinite(factor) && factor > 0 ? factor : Number.parseFloat(DEFAULT_VLLM_YARN_FACTOR), original_max_position_embeddings: Number.isFinite(originalMaxPositionEmbeddings) && originalMaxPositionEmbeddings > 0 ? originalMaxPositionEmbeddings : Number.parseInt(DEFAULT_VLLM_ORIGINAL_CONTEXT, 10), }, }) }, [maxModelLen, originalContext, yarnFactor]) const runCommand = useMemo(() => { const dockerArgs = [ "docker run -d", `--name ${shellQuote(containerName || DEFAULT_VLLM_CONTAINER)}`, "--runtime nvidia", `--gpus ${shellQuote(`"device=${gpuDevice || "0"}"`)}`, "--ipc=host", "--shm-size=64g", "--ulimit memlock=-1", "--ulimit stack=67108864", `-e CUDA_VISIBLE_DEVICES=${shellQuote(gpuDevice || "0")}`, "-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", allowLongMaxModelLen ? "-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1" : "", `-v "$HOME/.cache/huggingface:/root/.cache/huggingface"`, `-p ${shellQuote(hostPort || "8000")}:${shellQuote(containerPort || "8000")}`, "--restart unless-stopped", shellQuote(image || DEFAULT_VLLM_IMAGE), "python3 -m vllm.entrypoints.openai.api_server", `--model ${shellQuote(model || DEFAULT_VLLM_MODEL)}`, "--host 0.0.0.0", `--port ${shellQuote(containerPort || "8000")}`, `--tensor-parallel-size ${shellQuote(tensorParallelSize || "1")}`, enableYarnScaling ? `--hf-overrides ${shellQuote(hfOverridesJson)}` : "", `--max-model-len ${shellQuote(maxModelLen || DEFAULT_VLLM_MAX_MODEL_LEN)}`, `--gpu-memory-utilization ${shellQuote(gpuMemoryUtilization || "0.85")}`, `--dtype ${shellQuote(dtype || "bfloat16")}`, kvCacheDtype && kvCacheDtype !== "auto" ? `--kv-cache-dtype ${shellQuote(kvCacheDtype)}` : "", maxNumBatchedTokens ? `--max-num-batched-tokens ${shellQuote(maxNumBatchedTokens)}` : "", maxNumSeqs ? `--max-num-seqs ${shellQuote(maxNumSeqs)}` : "", enableChunkedPrefill ? "--enable-chunked-prefill" : "", enableAutoToolChoice ? "--enable-auto-tool-choice" : "", enableAutoToolChoice && toolCallParser ? `--tool-call-parser ${shellQuote(toolCallParser)}` : "", calculateKvScales && kvCacheDtype.startsWith("fp8") ? "--calculate-kv-scales" : "", ] return commandLine(dockerArgs) }, [ allowLongMaxModelLen, calculateKvScales, containerName, containerPort, dtype, enableAutoToolChoice, enableChunkedPrefill, enableYarnScaling, gpuDevice, gpuMemoryUtilization, hfOverridesJson, hostPort, image, kvCacheDtype, maxModelLen, maxNumBatchedTokens, maxNumSeqs, model, tensorParallelSize, toolCallParser, ]) async function copyRunCommand() { try { await navigator.clipboard.writeText(runCommand) setCopied(true) window.setTimeout(() => setCopied(false), 1600) } catch { setCopied(false) } } async function applyVllmContainer() { const target = containerName.trim() || DEFAULT_VLLM_CONTAINER if (!window.confirm(`Recreate vLLM container '${target}' with this configuration?`)) return try { setApplying(true) setRestartMessage("") const response = await fetch("/api/inference/vllm/apply", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ containerName: target, image, gpuDevice, hostPort, containerPort, model: model || DEFAULT_VLLM_MODEL, tensorParallelSize, maxModelLen, gpuMemoryUtilization, dtype, kvCacheDtype, maxNumBatchedTokens, maxNumSeqs, toolCallParser, allowLongMaxModelLen, enableYarnScaling, yarnFactor, originalContext, enableChunkedPrefill, enableAutoToolChoice, calculateKvScales, }), }) const data = await response.json() if (!response.ok) throw new Error(data.error || "Failed to apply vLLM configuration") setRestartMessage(`Recreated ${data.container || target}. Wait for the model to finish loading before sending requests.`) } catch (error) { setRestartMessage(error instanceof Error ? error.message : "Failed to apply vLLM configuration") } finally { setApplying(false) } } async function restartVllmContainer() { const target = containerName.trim() || DEFAULT_VLLM_CONTAINER if (!window.confirm(`Restart vLLM container '${target}'?`)) return try { setRestarting(true) setRestartMessage("") const response = await fetch("/api/inference/vllm/restart", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ containerName: target }), }) const data = await response.json() if (!response.ok) throw new Error(data.error || "Failed to restart vLLM container") setRestartMessage(`Restarted ${data.container || target}. Wait for the model to finish loading before sending requests.`) } catch (error) { setRestartMessage(error instanceof Error ? error.message : "Failed to restart vLLM container") } finally { setRestarting(false) } } return (
Input budget at 4096 output tokens: {maxInputWithDefaultOutput.toLocaleString()} tokens.
{restartMessage}
}Configure the OpenShell gateway route used by sandboxes through inference.local.
Sandbox Route
{gateway.configured ? gateway.provider : "not configured"}
{gateway.model || "No model selected"}
{gateway.timeout &&Timeout {gateway.timeout}
}System Route
{system.configured ? system.provider : "not configured"}
{system.model || "No model selected"}
{system.timeout &&Timeout {system.timeout}
}{ollamaMessage || "Polling local Ollama every 10 seconds."}
No local Ollama models reported.
)}{message}
}