diff --git a/app/api/inference/route.ts b/app/api/inference/route.ts index 32191f1..03031a7 100644 --- a/app/api/inference/route.ts +++ b/app/api/inference/route.ts @@ -6,7 +6,7 @@ import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands" const execFileAsync = promisify(execFile) -const PROVIDER_TYPES = new Set(["openai", "anthropic", "nvidia", "generic", "claude", "opencode", "codex", "copilot", "gitlab", "github", "outlook"]) +const PROVIDER_TYPES = new Set(["openai", "vllm", "anthropic", "nvidia", "generic", "claude", "opencode", "codex", "copilot", "gitlab", "github", "outlook"]) type InferenceRoute = { configured: boolean @@ -111,7 +111,7 @@ function isOllamaEndpoint(name: string, baseUrl: string) { } function normalizeProviderType(type: string, name: string, baseUrl: string) { - return isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) ? "openai" : type + return type === "vllm" || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) ? "openai" : type } function redactSecretOutput(value: string, secrets: string[] = []) { @@ -157,7 +157,7 @@ export async function POST(request: Request) { const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY" const makeActive = body?.makeActive !== false const system = body?.route === "system" || body?.system === true - const noVerify = body?.noVerify !== false || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) + const noVerify = body?.noVerify !== false || requestedType === "vllm" || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) const timeout = Number(body?.timeout ?? 0) if (makeActive && !model) throw new Error("model is required when making the endpoint active") diff --git a/app/api/inference/vllm/apply/route.ts b/app/api/inference/vllm/apply/route.ts new file mode 100644 index 0000000..8f76cc0 --- /dev/null +++ b/app/api/inference/vllm/apply/route.ts @@ -0,0 +1,226 @@ +import { NextResponse } from "next/server" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { HOST_PATH } from "@/app/lib/hostCommands" + +const execFileAsync = promisify(execFile) + +const DOCKER_BIN = process.env.DOCKER_BIN || "docker" +const DEFAULT_VLLM_CONTAINER = process.env.OPENSHELL_CONTROL_VLLM_CONTAINER || "vllm-bigboy" +const DEFAULT_VLLM_IMAGE = "nvcr.io/nvidia/vllm:26.03.post1-py3" +const DEFAULT_VLLM_MODEL = "vllm-local" +const DEFAULT_VLLM_MAX_MODEL_LEN = "32768" +const DEFAULT_VLLM_ORIGINAL_CONTEXT = 32768 +const DEFAULT_VLLM_YARN_FACTOR = 1 + +const DTYPES = new Set(["auto", "bfloat16", "float16", "float", "float32", "half"]) +const KV_CACHE_DTYPES = new Set(["auto", "bfloat16", "fp8", "fp8_ds_mla", "fp8_e4m3", "fp8_e5m2", "fp8_inc"]) +const TOOL_CALL_PARSERS = new Set(["", "hermes", "llama3_json", "mistral", "openai", "qwen3_coder"]) + +function configuredContainerAllowlist() { + return new Set([ + DEFAULT_VLLM_CONTAINER, + ...(process.env.OPENSHELL_CONTROL_VLLM_CONTAINERS || process.env.VLLM_CONTAINER_ALLOWLIST || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ]) +} + +function text(value: unknown, fallback: string) { + return typeof value === "string" && value.trim() ? value.trim() : fallback +} + +function bool(value: unknown, fallback: boolean) { + return typeof value === "boolean" ? value : fallback +} + +function validateContainerName(value: unknown) { + const container = text(value, DEFAULT_VLLM_CONTAINER) + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(container)) throw new Error("invalid vLLM container name") + const allowlist = configuredContainerAllowlist() + if (!allowlist.has(container) && !/^vllm(?:$|[_.-])/i.test(container)) { + throw new Error("vLLM container apply is limited to vLLM-named or allowlisted containers") + } + return container +} + +function validateImage(value: unknown) { + const image = text(value, DEFAULT_VLLM_IMAGE) + if (!/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,255}$/.test(image)) throw new Error("invalid vLLM image") + return image +} + +function validateModel(value: unknown) { + const model = text(value, DEFAULT_VLLM_MODEL) + if (!/^[a-zA-Z0-9][a-zA-Z0-9._/-]{0,255}$/.test(model)) throw new Error("invalid vLLM model id") + return model +} + +function validateGpuDevice(value: unknown) { + const gpuDevice = text(value, "0") + if (!/^\d+(?:,\d+)*$/.test(gpuDevice)) throw new Error("GPU device must be a comma-separated list of device ids") + return gpuDevice +} + +function validatePort(value: unknown, fallback: string) { + const port = Number.parseInt(text(value, fallback), 10) + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("port must be between 1 and 65535") + return String(port) +} + +function validateInteger(value: unknown, fallback: string, label: string, min: number, max: number) { + const parsed = Number.parseInt(text(value, fallback), 10) + if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new Error(`${label} must be between ${min} and ${max}`) + return String(parsed) +} + +function validateOptionalInteger(value: unknown, label: string, min: number, max: number) { + const raw = typeof value === "string" ? value.trim() : "" + if (!raw) return "" + const parsed = Number.parseInt(raw, 10) + if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new Error(`${label} must be between ${min} and ${max}`) + return String(parsed) +} + +function validateMemoryUtilization(value: unknown) { + const parsed = Number.parseFloat(text(value, "0.85")) + if (!Number.isFinite(parsed) || parsed < 0.1 || parsed > 0.99) throw new Error("GPU memory utilization must be between 0.1 and 0.99") + return String(parsed) +} + +function validateEnum(value: unknown, fallback: string, allowed: Set, label: string) { + const selected = text(value, fallback) + if (!allowed.has(selected)) throw new Error(`unsupported ${label}: ${selected}`) + return selected +} + +function validateYarnFactor(value: unknown) { + const parsed = Number.parseFloat(text(value, String(DEFAULT_VLLM_YARN_FACTOR))) + if (!Number.isFinite(parsed) || parsed < 1 || parsed > 16) throw new Error("YaRN factor must be between 1 and 16") + return parsed +} + +function dockerEnv() { + return { ...process.env, PATH: HOST_PATH } +} + +async function removeExistingContainer(container: string) { + try { + await execFileAsync(DOCKER_BIN, ["rm", "-f", container], { + env: dockerEnv(), + timeout: 120000, + maxBuffer: 4 * 1024 * 1024, + }) + } catch (error) { + const stderr = String((error as { stderr?: unknown }).stderr || "") + if (!/No such container/i.test(stderr)) throw error + } +} + +export async function POST(request: Request) { + try { + const body = await request.json().catch(() => ({})) + const container = validateContainerName(body?.containerName ?? body?.container) + const image = validateImage(body?.image) + const model = validateModel(body?.model) + const gpuDevice = validateGpuDevice(body?.gpuDevice) + const hostPort = validatePort(body?.hostPort, "8000") + const containerPort = validatePort(body?.containerPort, "8000") + const tensorParallelSize = validateInteger(body?.tensorParallelSize, "1", "tensor parallel size", 1, 64) + const maxModelLen = validateInteger(body?.maxModelLen, DEFAULT_VLLM_MAX_MODEL_LEN, "max model len", 1024, 262144) + const gpuMemoryUtilization = validateMemoryUtilization(body?.gpuMemoryUtilization) + const dtype = validateEnum(body?.dtype, "bfloat16", DTYPES, "dtype") + const kvCacheDtype = validateEnum(body?.kvCacheDtype, "fp8", KV_CACHE_DTYPES, "KV cache dtype") + const maxNumBatchedTokens = validateOptionalInteger(body?.maxNumBatchedTokens, "max batched tokens", 1, 262144) + const maxNumSeqs = validateOptionalInteger(body?.maxNumSeqs, "max seqs", 1, 4096) + const toolCallParser = validateEnum(body?.toolCallParser, "hermes", TOOL_CALL_PARSERS, "tool parser") + const allowLongMaxModelLen = bool(body?.allowLongMaxModelLen, false) + const enableYarnScaling = bool(body?.enableYarnScaling, false) + const yarnFactor = validateYarnFactor(body?.yarnFactor) + const originalContext = Number.parseInt( + validateInteger(body?.originalContext, String(DEFAULT_VLLM_ORIGINAL_CONTEXT), "original context", 1024, 262144), + 10, + ) + const enableChunkedPrefill = bool(body?.enableChunkedPrefill, true) + const enableAutoToolChoice = bool(body?.enableAutoToolChoice, true) + const calculateKvScales = bool(body?.calculateKvScales, false) + const hfCacheDir = `${process.env.HOME || "/root"}/.cache/huggingface` + + const runArgs = [ + "run", "-d", + "--name", container, + "--runtime", "nvidia", + "--gpus", `"device=${gpuDevice}"`, + "--ipc=host", + "--shm-size=64g", + "--ulimit", "memlock=-1", + "--ulimit", "stack=67108864", + "-e", `CUDA_VISIBLE_DEVICES=${gpuDevice}`, + "-e", "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", + ...(allowLongMaxModelLen ? ["-e", "VLLM_ALLOW_LONG_MAX_MODEL_LEN=1"] : []), + "-v", `${hfCacheDir}:/root/.cache/huggingface`, + "-p", `${hostPort}:${containerPort}`, + "--restart", "unless-stopped", + image, + "python3", "-m", "vllm.entrypoints.openai.api_server", + "--model", model, + "--host", "0.0.0.0", + "--port", containerPort, + "--tensor-parallel-size", tensorParallelSize, + ] + + if (enableYarnScaling) { + runArgs.push("--hf-overrides", JSON.stringify({ + max_position_embeddings: Number.parseInt(maxModelLen, 10), + rope_scaling: { + type: "yarn", + factor: yarnFactor, + original_max_position_embeddings: originalContext, + }, + })) + } + + runArgs.push( + "--max-model-len", maxModelLen, + "--gpu-memory-utilization", gpuMemoryUtilization, + "--dtype", dtype, + ) + + if (kvCacheDtype !== "auto") runArgs.push("--kv-cache-dtype", kvCacheDtype) + if (maxNumBatchedTokens) runArgs.push("--max-num-batched-tokens", maxNumBatchedTokens) + if (maxNumSeqs) runArgs.push("--max-num-seqs", maxNumSeqs) + if (enableChunkedPrefill) runArgs.push("--enable-chunked-prefill") + if (enableAutoToolChoice) { + runArgs.push("--enable-auto-tool-choice") + if (toolCallParser) runArgs.push("--tool-call-parser", toolCallParser) + } + if (calculateKvScales && kvCacheDtype.startsWith("fp8")) runArgs.push("--calculate-kv-scales") + + await removeExistingContainer(container) + const { stdout, stderr } = await execFileAsync(DOCKER_BIN, runArgs, { + env: dockerEnv(), + timeout: 120000, + maxBuffer: 4 * 1024 * 1024, + }) + + return NextResponse.json({ + ok: true, + applied: true, + container, + image, + model, + hostPort, + containerPort, + stdout: String(stdout).trim(), + stderr: String(stderr).trim(), + note: "vLLM container recreated. The model may take time to load before /v1/models is ready.", + }) + } catch (error) { + const message = error instanceof Error ? error.message : "Failed to apply vLLM container configuration" + return NextResponse.json( + { ok: false, applied: false, error: message }, + { status: /invalid|limited|required|must|unsupported|between/.test(message) ? 400 : 500 }, + ) + } +} diff --git a/app/api/inference/vllm/restart/route.ts b/app/api/inference/vllm/restart/route.ts new file mode 100644 index 0000000..79de7f1 --- /dev/null +++ b/app/api/inference/vllm/restart/route.ts @@ -0,0 +1,61 @@ +import { NextResponse } from "next/server" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { HOST_PATH } from "@/app/lib/hostCommands" + +const execFileAsync = promisify(execFile) +const DOCKER_BIN = process.env.DOCKER_BIN || "docker" +const DEFAULT_VLLM_CONTAINER = process.env.OPENSHELL_CONTROL_VLLM_CONTAINER || "vllm-bigboy" + +function configuredContainerAllowlist() { + return new Set([ + DEFAULT_VLLM_CONTAINER, + ...(process.env.OPENSHELL_CONTROL_VLLM_CONTAINERS || process.env.VLLM_CONTAINER_ALLOWLIST || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ]) +} + +function validateContainerName(value: unknown) { + const container = typeof value === "string" && value.trim() ? value.trim() : DEFAULT_VLLM_CONTAINER + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(container)) { + throw new Error("invalid vLLM container name") + } + const allowlist = configuredContainerAllowlist() + if (!allowlist.has(container) && !/^vllm(?:$|[_.-])/i.test(container)) { + throw new Error("vLLM container restart is limited to vLLM-named or allowlisted containers") + } + return container +} + +export async function POST(request: Request) { + try { + const body = await request.json().catch(() => ({})) + const container = validateContainerName(body?.containerName ?? body?.container) + const { stdout, stderr } = await execFileAsync(DOCKER_BIN, ["restart", container], { + env: { + ...process.env, + PATH: HOST_PATH, + }, + timeout: 120000, + maxBuffer: 4 * 1024 * 1024, + }) + + return NextResponse.json({ + ok: true, + restarted: true, + container, + stdout: String(stdout).trim(), + stderr: String(stderr).trim(), + note: "vLLM container restart requested. The model may take time to reload before /v1/models is ready.", + }) + } catch (error) { + const message = error instanceof Error ? error.message : "Failed to restart vLLM container" + return NextResponse.json({ + ok: false, + restarted: false, + error: message, + }, { status: /invalid|limited|required/.test(message) ? 400 : 500 }) + } +} diff --git a/app/components/InferenceEndpointPanel.tsx b/app/components/InferenceEndpointPanel.tsx index 741c47b..ae216c8 100644 --- a/app/components/InferenceEndpointPanel.tsx +++ b/app/components/InferenceEndpointPanel.tsx @@ -36,12 +36,33 @@ type OllamaModel = { const providerTypeOptions = [ { value: "openai", label: "OpenAI compatible" }, + { value: "vllm", label: "vLLM" }, { value: "anthropic", label: "Anthropic" }, { value: "generic", label: "Generic" }, ] const OLLAMA_PROVIDER_NAME = "ollama-local" const OLLAMA_BASE_URL = "http://host.docker.internal:11434/v1" +const DEFAULT_VLLM_CONTAINER = "vllm-bigboy" +const DEFAULT_VLLM_IMAGE = "nvcr.io/nvidia/vllm:26.03.post1-py3" +const DEFAULT_VLLM_MODEL = "vllm-local" +const DEFAULT_VLLM_MAX_MODEL_LEN = "32768" +const DEFAULT_VLLM_YARN_FACTOR = "1.0" +const DEFAULT_VLLM_ORIGINAL_CONTEXT = "32768" + +function isVllmEndpoint(name: string, baseUrl: string) { + return [name, baseUrl].some((value) => /vllm/i.test(value)) +} + +function shellQuote(value: string) { + const trimmed = value.trim() + if (!trimmed) return "''" + return /^[A-Za-z0-9_./:@%+=,-]+$/.test(trimmed) ? trimmed : `'${trimmed.replace(/'/g, "'\\''")}'` +} + +function commandLine(parts: string[]) { + return parts.filter(Boolean).join(" \\\n ") +} function emptyRoute(): InferenceRoute { return { configured: false, provider: null, model: null, version: null, timeout: null } @@ -51,6 +72,324 @@ function FieldLabel({ children }: { children: React.ReactNode }) { return } +function VllmAdvancedConfiguration({ + model, + port, +}: { + model: string + port: string +}) { + const [containerName, setContainerName] = useState(DEFAULT_VLLM_CONTAINER) + const [image, setImage] = useState(DEFAULT_VLLM_IMAGE) + const [gpuDevice, setGpuDevice] = useState("0") + const [hostPort, setHostPort] = useState("8000") + const [containerPort, setContainerPort] = useState("8000") + const [tensorParallelSize, setTensorParallelSize] = useState("1") + const [maxModelLen, setMaxModelLen] = useState(DEFAULT_VLLM_MAX_MODEL_LEN) + const [gpuMemoryUtilization, setGpuMemoryUtilization] = useState("0.85") + const [dtype, setDtype] = useState("bfloat16") + const [kvCacheDtype, setKvCacheDtype] = useState("fp8") + const [maxNumBatchedTokens, setMaxNumBatchedTokens] = useState("8192") + const [maxNumSeqs, setMaxNumSeqs] = useState("") + const [toolCallParser, setToolCallParser] = useState("hermes") + const [allowLongMaxModelLen, setAllowLongMaxModelLen] = useState(false) + const [enableYarnScaling, setEnableYarnScaling] = useState(false) + const [yarnFactor, setYarnFactor] = useState(DEFAULT_VLLM_YARN_FACTOR) + const [originalContext, setOriginalContext] = useState(DEFAULT_VLLM_ORIGINAL_CONTEXT) + const [enableChunkedPrefill, setEnableChunkedPrefill] = useState(true) + const [enableAutoToolChoice, setEnableAutoToolChoice] = useState(true) + const [calculateKvScales, setCalculateKvScales] = useState(false) + const [copied, setCopied] = useState(false) + const [applying, setApplying] = useState(false) + const [restarting, setRestarting] = useState(false) + const [restartMessage, setRestartMessage] = useState("") + + useEffect(() => { + if (port && hostPort === "8000") setHostPort(port) + if (port && containerPort === "8000") setContainerPort(port) + }, [containerPort, hostPort, port]) + + const maxInputWithDefaultOutput = useMemo(() => { + const parsed = Number.parseInt(maxModelLen, 10) + return Number.isFinite(parsed) ? Math.max(0, parsed - 4096) : 0 + }, [maxModelLen]) + + const hfOverridesJson = useMemo(() => { + const factor = Number.parseFloat(yarnFactor) + const originalMaxPositionEmbeddings = Number.parseInt(originalContext, 10) + const maxPositionEmbeddings = Number.parseInt(maxModelLen, 10) + return JSON.stringify({ + max_position_embeddings: Number.isFinite(maxPositionEmbeddings) && maxPositionEmbeddings > 0 + ? maxPositionEmbeddings + : Number.parseInt(DEFAULT_VLLM_MAX_MODEL_LEN, 10), + rope_scaling: { + type: "yarn", + factor: Number.isFinite(factor) && factor > 0 ? factor : Number.parseFloat(DEFAULT_VLLM_YARN_FACTOR), + original_max_position_embeddings: Number.isFinite(originalMaxPositionEmbeddings) && originalMaxPositionEmbeddings > 0 + ? originalMaxPositionEmbeddings + : Number.parseInt(DEFAULT_VLLM_ORIGINAL_CONTEXT, 10), + }, + }) + }, [maxModelLen, originalContext, yarnFactor]) + + const runCommand = useMemo(() => { + const dockerArgs = [ + "docker run -d", + `--name ${shellQuote(containerName || DEFAULT_VLLM_CONTAINER)}`, + "--runtime nvidia", + `--gpus ${shellQuote(`"device=${gpuDevice || "0"}"`)}`, + "--ipc=host", + "--shm-size=64g", + "--ulimit memlock=-1", + "--ulimit stack=67108864", + `-e CUDA_VISIBLE_DEVICES=${shellQuote(gpuDevice || "0")}`, + "-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", + allowLongMaxModelLen ? "-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1" : "", + `-v "$HOME/.cache/huggingface:/root/.cache/huggingface"`, + `-p ${shellQuote(hostPort || "8000")}:${shellQuote(containerPort || "8000")}`, + "--restart unless-stopped", + shellQuote(image || DEFAULT_VLLM_IMAGE), + "python3 -m vllm.entrypoints.openai.api_server", + `--model ${shellQuote(model || DEFAULT_VLLM_MODEL)}`, + "--host 0.0.0.0", + `--port ${shellQuote(containerPort || "8000")}`, + `--tensor-parallel-size ${shellQuote(tensorParallelSize || "1")}`, + enableYarnScaling ? `--hf-overrides ${shellQuote(hfOverridesJson)}` : "", + `--max-model-len ${shellQuote(maxModelLen || DEFAULT_VLLM_MAX_MODEL_LEN)}`, + `--gpu-memory-utilization ${shellQuote(gpuMemoryUtilization || "0.85")}`, + `--dtype ${shellQuote(dtype || "bfloat16")}`, + kvCacheDtype && kvCacheDtype !== "auto" ? `--kv-cache-dtype ${shellQuote(kvCacheDtype)}` : "", + maxNumBatchedTokens ? `--max-num-batched-tokens ${shellQuote(maxNumBatchedTokens)}` : "", + maxNumSeqs ? `--max-num-seqs ${shellQuote(maxNumSeqs)}` : "", + enableChunkedPrefill ? "--enable-chunked-prefill" : "", + enableAutoToolChoice ? "--enable-auto-tool-choice" : "", + enableAutoToolChoice && toolCallParser ? `--tool-call-parser ${shellQuote(toolCallParser)}` : "", + calculateKvScales && kvCacheDtype.startsWith("fp8") ? "--calculate-kv-scales" : "", + ] + return commandLine(dockerArgs) + }, [ + allowLongMaxModelLen, + calculateKvScales, + containerName, + containerPort, + dtype, + enableAutoToolChoice, + enableChunkedPrefill, + enableYarnScaling, + gpuDevice, + gpuMemoryUtilization, + hfOverridesJson, + hostPort, + image, + kvCacheDtype, + maxModelLen, + maxNumBatchedTokens, + maxNumSeqs, + model, + tensorParallelSize, + toolCallParser, + ]) + + async function copyRunCommand() { + try { + await navigator.clipboard.writeText(runCommand) + setCopied(true) + window.setTimeout(() => setCopied(false), 1600) + } catch { + setCopied(false) + } + } + + async function applyVllmContainer() { + const target = containerName.trim() || DEFAULT_VLLM_CONTAINER + if (!window.confirm(`Recreate vLLM container '${target}' with this configuration?`)) return + try { + setApplying(true) + setRestartMessage("") + const response = await fetch("/api/inference/vllm/apply", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + containerName: target, + image, + gpuDevice, + hostPort, + containerPort, + model: model || DEFAULT_VLLM_MODEL, + tensorParallelSize, + maxModelLen, + gpuMemoryUtilization, + dtype, + kvCacheDtype, + maxNumBatchedTokens, + maxNumSeqs, + toolCallParser, + allowLongMaxModelLen, + enableYarnScaling, + yarnFactor, + originalContext, + enableChunkedPrefill, + enableAutoToolChoice, + calculateKvScales, + }), + }) + const data = await response.json() + if (!response.ok) throw new Error(data.error || "Failed to apply vLLM configuration") + setRestartMessage(`Recreated ${data.container || target}. Wait for the model to finish loading before sending requests.`) + } catch (error) { + setRestartMessage(error instanceof Error ? error.message : "Failed to apply vLLM configuration") + } finally { + setApplying(false) + } + } + + async function restartVllmContainer() { + const target = containerName.trim() || DEFAULT_VLLM_CONTAINER + if (!window.confirm(`Restart vLLM container '${target}'?`)) return + try { + setRestarting(true) + setRestartMessage("") + const response = await fetch("/api/inference/vllm/restart", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ containerName: target }), + }) + const data = await response.json() + if (!response.ok) throw new Error(data.error || "Failed to restart vLLM container") + setRestartMessage(`Restarted ${data.container || target}. Wait for the model to finish loading before sending requests.`) + } catch (error) { + setRestartMessage(error instanceof Error ? error.message : "Failed to restart vLLM container") + } finally { + setRestarting(false) + } + } + + return ( +
+
+
+

vLLM Advanced Configuration

+

Input budget at 4096 output tokens: {maxInputWithDefaultOutput.toLocaleString()} tokens.

+
+
+ + + +
+
+ +
+
+ Container + setContainerName(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Image + setImage(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ GPU Device + setGpuDevice(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Host Port + setHostPort(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Container Port + setContainerPort(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Max Model Len + setMaxModelLen(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ GPU Memory + setGpuMemoryUtilization(event.target.value)} inputMode="decimal" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Tensor Parallel + setTensorParallelSize(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ dtype + +
+
+ KV Cache dtype + +
+
+ Tool Parser + +
+
+ Max Batched Tokens + setMaxNumBatchedTokens(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Max Seqs + setMaxNumSeqs(event.target.value)} inputMode="numeric" placeholder="auto" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ YaRN Factor + setYarnFactor(event.target.value)} inputMode="decimal" disabled={!enableYarnScaling} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)] disabled:opacity-50" /> +
+
+ Original Context + setOriginalContext(event.target.value)} inputMode="numeric" disabled={!enableYarnScaling} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)] disabled:opacity-50" /> +
+
+ +
+ + + + + +
+ +