Add vLLM advanced configuration

This commit is contained in:
mmckeen-nv
2026-05-12 16:06:45 -07:00
parent c197d9eb28
commit 2f51629324
5 changed files with 680 additions and 7 deletions
+3 -3
View File
@@ -6,7 +6,7 @@ import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
const execFileAsync = promisify(execFile) const execFileAsync = promisify(execFile)
const PROVIDER_TYPES = new Set(["openai", "anthropic", "nvidia", "generic", "claude", "opencode", "codex", "copilot", "gitlab", "github", "outlook"]) const PROVIDER_TYPES = new Set(["openai", "vllm", "anthropic", "nvidia", "generic", "claude", "opencode", "codex", "copilot", "gitlab", "github", "outlook"])
type InferenceRoute = { type InferenceRoute = {
configured: boolean configured: boolean
@@ -111,7 +111,7 @@ function isOllamaEndpoint(name: string, baseUrl: string) {
} }
function normalizeProviderType(type: string, name: string, baseUrl: string) { function normalizeProviderType(type: string, name: string, baseUrl: string) {
return isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) ? "openai" : type return type === "vllm" || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) ? "openai" : type
} }
function redactSecretOutput(value: string, secrets: string[] = []) { function redactSecretOutput(value: string, secrets: string[] = []) {
@@ -157,7 +157,7 @@ export async function POST(request: Request) {
const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY" const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY"
const makeActive = body?.makeActive !== false const makeActive = body?.makeActive !== false
const system = body?.route === "system" || body?.system === true const system = body?.route === "system" || body?.system === true
const noVerify = body?.noVerify !== false || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) const noVerify = body?.noVerify !== false || requestedType === "vllm" || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl)
const timeout = Number(body?.timeout ?? 0) const timeout = Number(body?.timeout ?? 0)
if (makeActive && !model) throw new Error("model is required when making the endpoint active") if (makeActive && !model) throw new Error("model is required when making the endpoint active")
+226
View File
@@ -0,0 +1,226 @@
import { NextResponse } from "next/server"
import { execFile } from "node:child_process"
import { promisify } from "node:util"
import { HOST_PATH } from "@/app/lib/hostCommands"
const execFileAsync = promisify(execFile)
const DOCKER_BIN = process.env.DOCKER_BIN || "docker"
const DEFAULT_VLLM_CONTAINER = process.env.OPENSHELL_CONTROL_VLLM_CONTAINER || "vllm-bigboy"
const DEFAULT_VLLM_IMAGE = "nvcr.io/nvidia/vllm:26.03.post1-py3"
const DEFAULT_VLLM_MODEL = "vllm-local"
const DEFAULT_VLLM_MAX_MODEL_LEN = "32768"
const DEFAULT_VLLM_ORIGINAL_CONTEXT = 32768
const DEFAULT_VLLM_YARN_FACTOR = 1
const DTYPES = new Set(["auto", "bfloat16", "float16", "float", "float32", "half"])
const KV_CACHE_DTYPES = new Set(["auto", "bfloat16", "fp8", "fp8_ds_mla", "fp8_e4m3", "fp8_e5m2", "fp8_inc"])
const TOOL_CALL_PARSERS = new Set(["", "hermes", "llama3_json", "mistral", "openai", "qwen3_coder"])
function configuredContainerAllowlist() {
return new Set([
DEFAULT_VLLM_CONTAINER,
...(process.env.OPENSHELL_CONTROL_VLLM_CONTAINERS || process.env.VLLM_CONTAINER_ALLOWLIST || "")
.split(",")
.map((item) => item.trim())
.filter(Boolean),
])
}
function text(value: unknown, fallback: string) {
return typeof value === "string" && value.trim() ? value.trim() : fallback
}
function bool(value: unknown, fallback: boolean) {
return typeof value === "boolean" ? value : fallback
}
function validateContainerName(value: unknown) {
const container = text(value, DEFAULT_VLLM_CONTAINER)
if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(container)) throw new Error("invalid vLLM container name")
const allowlist = configuredContainerAllowlist()
if (!allowlist.has(container) && !/^vllm(?:$|[_.-])/i.test(container)) {
throw new Error("vLLM container apply is limited to vLLM-named or allowlisted containers")
}
return container
}
function validateImage(value: unknown) {
const image = text(value, DEFAULT_VLLM_IMAGE)
if (!/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,255}$/.test(image)) throw new Error("invalid vLLM image")
return image
}
function validateModel(value: unknown) {
const model = text(value, DEFAULT_VLLM_MODEL)
if (!/^[a-zA-Z0-9][a-zA-Z0-9._/-]{0,255}$/.test(model)) throw new Error("invalid vLLM model id")
return model
}
function validateGpuDevice(value: unknown) {
const gpuDevice = text(value, "0")
if (!/^\d+(?:,\d+)*$/.test(gpuDevice)) throw new Error("GPU device must be a comma-separated list of device ids")
return gpuDevice
}
function validatePort(value: unknown, fallback: string) {
const port = Number.parseInt(text(value, fallback), 10)
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("port must be between 1 and 65535")
return String(port)
}
function validateInteger(value: unknown, fallback: string, label: string, min: number, max: number) {
const parsed = Number.parseInt(text(value, fallback), 10)
if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new Error(`${label} must be between ${min} and ${max}`)
return String(parsed)
}
function validateOptionalInteger(value: unknown, label: string, min: number, max: number) {
const raw = typeof value === "string" ? value.trim() : ""
if (!raw) return ""
const parsed = Number.parseInt(raw, 10)
if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new Error(`${label} must be between ${min} and ${max}`)
return String(parsed)
}
function validateMemoryUtilization(value: unknown) {
const parsed = Number.parseFloat(text(value, "0.85"))
if (!Number.isFinite(parsed) || parsed < 0.1 || parsed > 0.99) throw new Error("GPU memory utilization must be between 0.1 and 0.99")
return String(parsed)
}
function validateEnum(value: unknown, fallback: string, allowed: Set<string>, label: string) {
const selected = text(value, fallback)
if (!allowed.has(selected)) throw new Error(`unsupported ${label}: ${selected}`)
return selected
}
function validateYarnFactor(value: unknown) {
const parsed = Number.parseFloat(text(value, String(DEFAULT_VLLM_YARN_FACTOR)))
if (!Number.isFinite(parsed) || parsed < 1 || parsed > 16) throw new Error("YaRN factor must be between 1 and 16")
return parsed
}
function dockerEnv() {
return { ...process.env, PATH: HOST_PATH }
}
async function removeExistingContainer(container: string) {
try {
await execFileAsync(DOCKER_BIN, ["rm", "-f", container], {
env: dockerEnv(),
timeout: 120000,
maxBuffer: 4 * 1024 * 1024,
})
} catch (error) {
const stderr = String((error as { stderr?: unknown }).stderr || "")
if (!/No such container/i.test(stderr)) throw error
}
}
export async function POST(request: Request) {
try {
const body = await request.json().catch(() => ({}))
const container = validateContainerName(body?.containerName ?? body?.container)
const image = validateImage(body?.image)
const model = validateModel(body?.model)
const gpuDevice = validateGpuDevice(body?.gpuDevice)
const hostPort = validatePort(body?.hostPort, "8000")
const containerPort = validatePort(body?.containerPort, "8000")
const tensorParallelSize = validateInteger(body?.tensorParallelSize, "1", "tensor parallel size", 1, 64)
const maxModelLen = validateInteger(body?.maxModelLen, DEFAULT_VLLM_MAX_MODEL_LEN, "max model len", 1024, 262144)
const gpuMemoryUtilization = validateMemoryUtilization(body?.gpuMemoryUtilization)
const dtype = validateEnum(body?.dtype, "bfloat16", DTYPES, "dtype")
const kvCacheDtype = validateEnum(body?.kvCacheDtype, "fp8", KV_CACHE_DTYPES, "KV cache dtype")
const maxNumBatchedTokens = validateOptionalInteger(body?.maxNumBatchedTokens, "max batched tokens", 1, 262144)
const maxNumSeqs = validateOptionalInteger(body?.maxNumSeqs, "max seqs", 1, 4096)
const toolCallParser = validateEnum(body?.toolCallParser, "hermes", TOOL_CALL_PARSERS, "tool parser")
const allowLongMaxModelLen = bool(body?.allowLongMaxModelLen, false)
const enableYarnScaling = bool(body?.enableYarnScaling, false)
const yarnFactor = validateYarnFactor(body?.yarnFactor)
const originalContext = Number.parseInt(
validateInteger(body?.originalContext, String(DEFAULT_VLLM_ORIGINAL_CONTEXT), "original context", 1024, 262144),
10,
)
const enableChunkedPrefill = bool(body?.enableChunkedPrefill, true)
const enableAutoToolChoice = bool(body?.enableAutoToolChoice, true)
const calculateKvScales = bool(body?.calculateKvScales, false)
const hfCacheDir = `${process.env.HOME || "/root"}/.cache/huggingface`
const runArgs = [
"run", "-d",
"--name", container,
"--runtime", "nvidia",
"--gpus", `"device=${gpuDevice}"`,
"--ipc=host",
"--shm-size=64g",
"--ulimit", "memlock=-1",
"--ulimit", "stack=67108864",
"-e", `CUDA_VISIBLE_DEVICES=${gpuDevice}`,
"-e", "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
...(allowLongMaxModelLen ? ["-e", "VLLM_ALLOW_LONG_MAX_MODEL_LEN=1"] : []),
"-v", `${hfCacheDir}:/root/.cache/huggingface`,
"-p", `${hostPort}:${containerPort}`,
"--restart", "unless-stopped",
image,
"python3", "-m", "vllm.entrypoints.openai.api_server",
"--model", model,
"--host", "0.0.0.0",
"--port", containerPort,
"--tensor-parallel-size", tensorParallelSize,
]
if (enableYarnScaling) {
runArgs.push("--hf-overrides", JSON.stringify({
max_position_embeddings: Number.parseInt(maxModelLen, 10),
rope_scaling: {
type: "yarn",
factor: yarnFactor,
original_max_position_embeddings: originalContext,
},
}))
}
runArgs.push(
"--max-model-len", maxModelLen,
"--gpu-memory-utilization", gpuMemoryUtilization,
"--dtype", dtype,
)
if (kvCacheDtype !== "auto") runArgs.push("--kv-cache-dtype", kvCacheDtype)
if (maxNumBatchedTokens) runArgs.push("--max-num-batched-tokens", maxNumBatchedTokens)
if (maxNumSeqs) runArgs.push("--max-num-seqs", maxNumSeqs)
if (enableChunkedPrefill) runArgs.push("--enable-chunked-prefill")
if (enableAutoToolChoice) {
runArgs.push("--enable-auto-tool-choice")
if (toolCallParser) runArgs.push("--tool-call-parser", toolCallParser)
}
if (calculateKvScales && kvCacheDtype.startsWith("fp8")) runArgs.push("--calculate-kv-scales")
await removeExistingContainer(container)
const { stdout, stderr } = await execFileAsync(DOCKER_BIN, runArgs, {
env: dockerEnv(),
timeout: 120000,
maxBuffer: 4 * 1024 * 1024,
})
return NextResponse.json({
ok: true,
applied: true,
container,
image,
model,
hostPort,
containerPort,
stdout: String(stdout).trim(),
stderr: String(stderr).trim(),
note: "vLLM container recreated. The model may take time to load before /v1/models is ready.",
})
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to apply vLLM container configuration"
return NextResponse.json(
{ ok: false, applied: false, error: message },
{ status: /invalid|limited|required|must|unsupported|between/.test(message) ? 400 : 500 },
)
}
}
+61
View File
@@ -0,0 +1,61 @@
import { NextResponse } from "next/server"
import { execFile } from "node:child_process"
import { promisify } from "node:util"
import { HOST_PATH } from "@/app/lib/hostCommands"
const execFileAsync = promisify(execFile)
const DOCKER_BIN = process.env.DOCKER_BIN || "docker"
const DEFAULT_VLLM_CONTAINER = process.env.OPENSHELL_CONTROL_VLLM_CONTAINER || "vllm-bigboy"
function configuredContainerAllowlist() {
return new Set([
DEFAULT_VLLM_CONTAINER,
...(process.env.OPENSHELL_CONTROL_VLLM_CONTAINERS || process.env.VLLM_CONTAINER_ALLOWLIST || "")
.split(",")
.map((item) => item.trim())
.filter(Boolean),
])
}
function validateContainerName(value: unknown) {
const container = typeof value === "string" && value.trim() ? value.trim() : DEFAULT_VLLM_CONTAINER
if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(container)) {
throw new Error("invalid vLLM container name")
}
const allowlist = configuredContainerAllowlist()
if (!allowlist.has(container) && !/^vllm(?:$|[_.-])/i.test(container)) {
throw new Error("vLLM container restart is limited to vLLM-named or allowlisted containers")
}
return container
}
export async function POST(request: Request) {
try {
const body = await request.json().catch(() => ({}))
const container = validateContainerName(body?.containerName ?? body?.container)
const { stdout, stderr } = await execFileAsync(DOCKER_BIN, ["restart", container], {
env: {
...process.env,
PATH: HOST_PATH,
},
timeout: 120000,
maxBuffer: 4 * 1024 * 1024,
})
return NextResponse.json({
ok: true,
restarted: true,
container,
stdout: String(stdout).trim(),
stderr: String(stderr).trim(),
note: "vLLM container restart requested. The model may take time to reload before /v1/models is ready.",
})
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to restart vLLM container"
return NextResponse.json({
ok: false,
restarted: false,
error: message,
}, { status: /invalid|limited|required/.test(message) ? 400 : 500 })
}
}
+355 -3
View File
@@ -36,12 +36,33 @@ type OllamaModel = {
const providerTypeOptions = [ const providerTypeOptions = [
{ value: "openai", label: "OpenAI compatible" }, { value: "openai", label: "OpenAI compatible" },
{ value: "vllm", label: "vLLM" },
{ value: "anthropic", label: "Anthropic" }, { value: "anthropic", label: "Anthropic" },
{ value: "generic", label: "Generic" }, { value: "generic", label: "Generic" },
] ]
const OLLAMA_PROVIDER_NAME = "ollama-local" const OLLAMA_PROVIDER_NAME = "ollama-local"
const OLLAMA_BASE_URL = "http://host.docker.internal:11434/v1" const OLLAMA_BASE_URL = "http://host.docker.internal:11434/v1"
const DEFAULT_VLLM_CONTAINER = "vllm-bigboy"
const DEFAULT_VLLM_IMAGE = "nvcr.io/nvidia/vllm:26.03.post1-py3"
const DEFAULT_VLLM_MODEL = "vllm-local"
const DEFAULT_VLLM_MAX_MODEL_LEN = "32768"
const DEFAULT_VLLM_YARN_FACTOR = "1.0"
const DEFAULT_VLLM_ORIGINAL_CONTEXT = "32768"
function isVllmEndpoint(name: string, baseUrl: string) {
return [name, baseUrl].some((value) => /vllm/i.test(value))
}
function shellQuote(value: string) {
const trimmed = value.trim()
if (!trimmed) return "''"
return /^[A-Za-z0-9_./:@%+=,-]+$/.test(trimmed) ? trimmed : `'${trimmed.replace(/'/g, "'\\''")}'`
}
function commandLine(parts: string[]) {
return parts.filter(Boolean).join(" \\\n ")
}
function emptyRoute(): InferenceRoute { function emptyRoute(): InferenceRoute {
return { configured: false, provider: null, model: null, version: null, timeout: null } return { configured: false, provider: null, model: null, version: null, timeout: null }
@@ -51,6 +72,324 @@ function FieldLabel({ children }: { children: React.ReactNode }) {
return <label className="text-[10px] uppercase tracking-wider text-[var(--foreground-dim)]">{children}</label> return <label className="text-[10px] uppercase tracking-wider text-[var(--foreground-dim)]">{children}</label>
} }
function VllmAdvancedConfiguration({
model,
port,
}: {
model: string
port: string
}) {
const [containerName, setContainerName] = useState(DEFAULT_VLLM_CONTAINER)
const [image, setImage] = useState(DEFAULT_VLLM_IMAGE)
const [gpuDevice, setGpuDevice] = useState("0")
const [hostPort, setHostPort] = useState("8000")
const [containerPort, setContainerPort] = useState("8000")
const [tensorParallelSize, setTensorParallelSize] = useState("1")
const [maxModelLen, setMaxModelLen] = useState(DEFAULT_VLLM_MAX_MODEL_LEN)
const [gpuMemoryUtilization, setGpuMemoryUtilization] = useState("0.85")
const [dtype, setDtype] = useState("bfloat16")
const [kvCacheDtype, setKvCacheDtype] = useState("fp8")
const [maxNumBatchedTokens, setMaxNumBatchedTokens] = useState("8192")
const [maxNumSeqs, setMaxNumSeqs] = useState("")
const [toolCallParser, setToolCallParser] = useState("hermes")
const [allowLongMaxModelLen, setAllowLongMaxModelLen] = useState(false)
const [enableYarnScaling, setEnableYarnScaling] = useState(false)
const [yarnFactor, setYarnFactor] = useState(DEFAULT_VLLM_YARN_FACTOR)
const [originalContext, setOriginalContext] = useState(DEFAULT_VLLM_ORIGINAL_CONTEXT)
const [enableChunkedPrefill, setEnableChunkedPrefill] = useState(true)
const [enableAutoToolChoice, setEnableAutoToolChoice] = useState(true)
const [calculateKvScales, setCalculateKvScales] = useState(false)
const [copied, setCopied] = useState(false)
const [applying, setApplying] = useState(false)
const [restarting, setRestarting] = useState(false)
const [restartMessage, setRestartMessage] = useState("")
useEffect(() => {
if (port && hostPort === "8000") setHostPort(port)
if (port && containerPort === "8000") setContainerPort(port)
}, [containerPort, hostPort, port])
const maxInputWithDefaultOutput = useMemo(() => {
const parsed = Number.parseInt(maxModelLen, 10)
return Number.isFinite(parsed) ? Math.max(0, parsed - 4096) : 0
}, [maxModelLen])
const hfOverridesJson = useMemo(() => {
const factor = Number.parseFloat(yarnFactor)
const originalMaxPositionEmbeddings = Number.parseInt(originalContext, 10)
const maxPositionEmbeddings = Number.parseInt(maxModelLen, 10)
return JSON.stringify({
max_position_embeddings: Number.isFinite(maxPositionEmbeddings) && maxPositionEmbeddings > 0
? maxPositionEmbeddings
: Number.parseInt(DEFAULT_VLLM_MAX_MODEL_LEN, 10),
rope_scaling: {
type: "yarn",
factor: Number.isFinite(factor) && factor > 0 ? factor : Number.parseFloat(DEFAULT_VLLM_YARN_FACTOR),
original_max_position_embeddings: Number.isFinite(originalMaxPositionEmbeddings) && originalMaxPositionEmbeddings > 0
? originalMaxPositionEmbeddings
: Number.parseInt(DEFAULT_VLLM_ORIGINAL_CONTEXT, 10),
},
})
}, [maxModelLen, originalContext, yarnFactor])
const runCommand = useMemo(() => {
const dockerArgs = [
"docker run -d",
`--name ${shellQuote(containerName || DEFAULT_VLLM_CONTAINER)}`,
"--runtime nvidia",
`--gpus ${shellQuote(`"device=${gpuDevice || "0"}"`)}`,
"--ipc=host",
"--shm-size=64g",
"--ulimit memlock=-1",
"--ulimit stack=67108864",
`-e CUDA_VISIBLE_DEVICES=${shellQuote(gpuDevice || "0")}`,
"-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
allowLongMaxModelLen ? "-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1" : "",
`-v "$HOME/.cache/huggingface:/root/.cache/huggingface"`,
`-p ${shellQuote(hostPort || "8000")}:${shellQuote(containerPort || "8000")}`,
"--restart unless-stopped",
shellQuote(image || DEFAULT_VLLM_IMAGE),
"python3 -m vllm.entrypoints.openai.api_server",
`--model ${shellQuote(model || DEFAULT_VLLM_MODEL)}`,
"--host 0.0.0.0",
`--port ${shellQuote(containerPort || "8000")}`,
`--tensor-parallel-size ${shellQuote(tensorParallelSize || "1")}`,
enableYarnScaling ? `--hf-overrides ${shellQuote(hfOverridesJson)}` : "",
`--max-model-len ${shellQuote(maxModelLen || DEFAULT_VLLM_MAX_MODEL_LEN)}`,
`--gpu-memory-utilization ${shellQuote(gpuMemoryUtilization || "0.85")}`,
`--dtype ${shellQuote(dtype || "bfloat16")}`,
kvCacheDtype && kvCacheDtype !== "auto" ? `--kv-cache-dtype ${shellQuote(kvCacheDtype)}` : "",
maxNumBatchedTokens ? `--max-num-batched-tokens ${shellQuote(maxNumBatchedTokens)}` : "",
maxNumSeqs ? `--max-num-seqs ${shellQuote(maxNumSeqs)}` : "",
enableChunkedPrefill ? "--enable-chunked-prefill" : "",
enableAutoToolChoice ? "--enable-auto-tool-choice" : "",
enableAutoToolChoice && toolCallParser ? `--tool-call-parser ${shellQuote(toolCallParser)}` : "",
calculateKvScales && kvCacheDtype.startsWith("fp8") ? "--calculate-kv-scales" : "",
]
return commandLine(dockerArgs)
}, [
allowLongMaxModelLen,
calculateKvScales,
containerName,
containerPort,
dtype,
enableAutoToolChoice,
enableChunkedPrefill,
enableYarnScaling,
gpuDevice,
gpuMemoryUtilization,
hfOverridesJson,
hostPort,
image,
kvCacheDtype,
maxModelLen,
maxNumBatchedTokens,
maxNumSeqs,
model,
tensorParallelSize,
toolCallParser,
])
async function copyRunCommand() {
try {
await navigator.clipboard.writeText(runCommand)
setCopied(true)
window.setTimeout(() => setCopied(false), 1600)
} catch {
setCopied(false)
}
}
async function applyVllmContainer() {
const target = containerName.trim() || DEFAULT_VLLM_CONTAINER
if (!window.confirm(`Recreate vLLM container '${target}' with this configuration?`)) return
try {
setApplying(true)
setRestartMessage("")
const response = await fetch("/api/inference/vllm/apply", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
containerName: target,
image,
gpuDevice,
hostPort,
containerPort,
model: model || DEFAULT_VLLM_MODEL,
tensorParallelSize,
maxModelLen,
gpuMemoryUtilization,
dtype,
kvCacheDtype,
maxNumBatchedTokens,
maxNumSeqs,
toolCallParser,
allowLongMaxModelLen,
enableYarnScaling,
yarnFactor,
originalContext,
enableChunkedPrefill,
enableAutoToolChoice,
calculateKvScales,
}),
})
const data = await response.json()
if (!response.ok) throw new Error(data.error || "Failed to apply vLLM configuration")
setRestartMessage(`Recreated ${data.container || target}. Wait for the model to finish loading before sending requests.`)
} catch (error) {
setRestartMessage(error instanceof Error ? error.message : "Failed to apply vLLM configuration")
} finally {
setApplying(false)
}
}
async function restartVllmContainer() {
const target = containerName.trim() || DEFAULT_VLLM_CONTAINER
if (!window.confirm(`Restart vLLM container '${target}'?`)) return
try {
setRestarting(true)
setRestartMessage("")
const response = await fetch("/api/inference/vllm/restart", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ containerName: target }),
})
const data = await response.json()
if (!response.ok) throw new Error(data.error || "Failed to restart vLLM container")
setRestartMessage(`Restarted ${data.container || target}. Wait for the model to finish loading before sending requests.`)
} catch (error) {
setRestartMessage(error instanceof Error ? error.message : "Failed to restart vLLM container")
} finally {
setRestarting(false)
}
}
return (
<div className="rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] p-4">
<div className="flex items-start justify-between gap-4">
<div>
<h3 className="text-xs uppercase tracking-wider text-[var(--foreground)]">vLLM Advanced Configuration</h3>
<p className="mt-1 text-xs text-[var(--foreground-dim)]">Input budget at 4096 output tokens: {maxInputWithDefaultOutput.toLocaleString()} tokens.</p>
</div>
<div className="flex items-center gap-2">
<button type="button" onClick={applyVllmContainer} disabled={applying || restarting} className="px-3 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
{applying ? "Applying" : "Apply"}
</button>
<button type="button" onClick={restartVllmContainer} disabled={restarting || applying} className="px-3 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
{restarting ? "Restarting" : "Restart"}
</button>
<button type="button" onClick={copyRunCommand} className="px-3 py-2 rounded-sm bg-[var(--background)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:bg-[var(--background-panel)]">
{copied ? "Copied" : "Copy"}
</button>
</div>
</div>
<div className="mt-4 grid grid-cols-1 md:grid-cols-3 gap-4">
<div className="space-y-2">
<FieldLabel>Container</FieldLabel>
<input value={containerName} onChange={(event) => setContainerName(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2 md:col-span-2">
<FieldLabel>Image</FieldLabel>
<input value={image} onChange={(event) => setImage(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>GPU Device</FieldLabel>
<input value={gpuDevice} onChange={(event) => setGpuDevice(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>Host Port</FieldLabel>
<input value={hostPort} onChange={(event) => setHostPort(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>Container Port</FieldLabel>
<input value={containerPort} onChange={(event) => setContainerPort(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>Max Model Len</FieldLabel>
<input value={maxModelLen} onChange={(event) => setMaxModelLen(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>GPU Memory</FieldLabel>
<input value={gpuMemoryUtilization} onChange={(event) => setGpuMemoryUtilization(event.target.value)} inputMode="decimal" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>Tensor Parallel</FieldLabel>
<input value={tensorParallelSize} onChange={(event) => setTensorParallelSize(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>dtype</FieldLabel>
<select value={dtype} onChange={(event) => setDtype(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]">
<option value="bfloat16">bfloat16</option>
<option value="float16">float16</option>
<option value="auto">auto</option>
</select>
</div>
<div className="space-y-2">
<FieldLabel>KV Cache dtype</FieldLabel>
<select value={kvCacheDtype} onChange={(event) => setKvCacheDtype(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]">
<option value="fp8">fp8</option>
<option value="fp8_e4m3">fp8_e4m3</option>
<option value="fp8_e5m2">fp8_e5m2</option>
<option value="auto">auto</option>
</select>
</div>
<div className="space-y-2">
<FieldLabel>Tool Parser</FieldLabel>
<select value={toolCallParser} onChange={(event) => setToolCallParser(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]">
<option value="hermes">hermes</option>
<option value="llama3_json">llama3_json</option>
<option value="mistral">mistral</option>
<option value="">none</option>
</select>
</div>
<div className="space-y-2">
<FieldLabel>Max Batched Tokens</FieldLabel>
<input value={maxNumBatchedTokens} onChange={(event) => setMaxNumBatchedTokens(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>Max Seqs</FieldLabel>
<input value={maxNumSeqs} onChange={(event) => setMaxNumSeqs(event.target.value)} inputMode="numeric" placeholder="auto" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
</div>
<div className="space-y-2">
<FieldLabel>YaRN Factor</FieldLabel>
<input value={yarnFactor} onChange={(event) => setYarnFactor(event.target.value)} inputMode="decimal" disabled={!enableYarnScaling} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)] disabled:opacity-50" />
</div>
<div className="space-y-2">
<FieldLabel>Original Context</FieldLabel>
<input value={originalContext} onChange={(event) => setOriginalContext(event.target.value)} inputMode="numeric" disabled={!enableYarnScaling} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)] disabled:opacity-50" />
</div>
</div>
<div className="mt-4 grid grid-cols-1 md:grid-cols-3 gap-3">
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
<input type="checkbox" checked={enableYarnScaling} onChange={(event) => setEnableYarnScaling(event.target.checked)} />
YaRN/RoPE scaling
</label>
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
<input type="checkbox" checked={allowLongMaxModelLen} onChange={(event) => setAllowLongMaxModelLen(event.target.checked)} />
Allow long max len
</label>
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
<input type="checkbox" checked={enableChunkedPrefill} onChange={(event) => setEnableChunkedPrefill(event.target.checked)} />
Chunked prefill
</label>
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
<input type="checkbox" checked={enableAutoToolChoice} onChange={(event) => setEnableAutoToolChoice(event.target.checked)} />
Auto tool choice
</label>
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
<input type="checkbox" checked={calculateKvScales} onChange={(event) => setCalculateKvScales(event.target.checked)} />
Calculate KV scales
</label>
</div>
<textarea readOnly value={runCommand} className="mt-4 min-h-[240px] w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] p-3 text-[11px] font-mono leading-5 text-[var(--foreground)] focus:outline-none" />
{restartMessage && <p className="mt-3 text-xs text-[var(--foreground-dim)]">{restartMessage}</p>}
</div>
)
}
export default function InferenceEndpointPanel() { export default function InferenceEndpointPanel() {
const [loading, setLoading] = useState(true) const [loading, setLoading] = useState(true)
const [saving, setSaving] = useState(false) const [saving, setSaving] = useState(false)
@@ -77,6 +416,14 @@ export default function InferenceEndpointPanel() {
[providers, gateway.provider] [providers, gateway.provider]
) )
const isOllamaEndpoint = name.toLowerCase().includes("ollama") || baseUrl.includes("11434") const isOllamaEndpoint = name.toLowerCase().includes("ollama") || baseUrl.includes("11434")
const isVllmEndpointSelected = type === "vllm"
const vllmPort = useMemo(() => {
try {
return new URL(baseUrl).port || "8000"
} catch {
return "8000"
}
}, [baseUrl])
async function load() { async function load() {
try { try {
@@ -90,6 +437,7 @@ export default function InferenceEndpointPanel() {
setProviders(Array.isArray(data.providers) ? data.providers : []) setProviders(Array.isArray(data.providers) ? data.providers : [])
if (data.gateway?.provider) setName(data.gateway.provider) if (data.gateway?.provider) setName(data.gateway.provider)
if (data.gateway?.model) setModel(data.gateway.model) if (data.gateway?.model) setModel(data.gateway.model)
if (data.gateway?.provider && isVllmEndpoint(data.gateway.provider, "")) setType("vllm")
} catch (error) { } catch (error) {
setMessage(error instanceof Error ? error.message : "Failed to load inference configuration") setMessage(error instanceof Error ? error.message : "Failed to load inference configuration")
} finally { } finally {
@@ -145,7 +493,7 @@ export default function InferenceEndpointPanel() {
function selectProvider(provider: ProviderSummary) { function selectProvider(provider: ProviderSummary) {
if (!provider.name) return if (!provider.name) return
setName(provider.name) setName(provider.name)
setType(provider.type || "openai") setType(isVllmEndpoint(provider.name, "") ? "vllm" : provider.type || "openai")
setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY") setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY")
if (provider.name.toLowerCase().includes("ollama")) { if (provider.name.toLowerCase().includes("ollama")) {
setBaseUrl(OLLAMA_BASE_URL) setBaseUrl(OLLAMA_BASE_URL)
@@ -156,8 +504,8 @@ export default function InferenceEndpointPanel() {
function useVllmPreset() { function useVllmPreset() {
setName("vllm-local") setName("vllm-local")
setType("openai") setType("vllm")
setModel("vllm-local") setModel(DEFAULT_VLLM_MODEL)
setBaseUrl("http://host.docker.internal:8000/v1") setBaseUrl("http://host.docker.internal:8000/v1")
setCredentialKey("OPENAI_API_KEY") setCredentialKey("OPENAI_API_KEY")
setApiKey("") setApiKey("")
@@ -343,6 +691,10 @@ export default function InferenceEndpointPanel() {
Skip provider verification while saving Skip provider verification while saving
</label> </label>
{isVllmEndpointSelected && (
<VllmAdvancedConfiguration model={model} port={vllmPort} />
)}
<div className="flex items-center gap-3"> <div className="flex items-center gap-3">
<button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50"> <button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
{saving ? "Saving..." : "Save Endpoint"} {saving ? "Saving..." : "Save Endpoint"}
+35 -1
View File
@@ -8,6 +8,8 @@ const deleteRoutePath = path.join(root, 'app/api/sandbox/delete/route.ts')
const hostCommandsPath = path.join(root, 'app/lib/hostCommands.ts') const hostCommandsPath = path.join(root, 'app/lib/hostCommands.ts')
const telemetryRoutePath = path.join(root, 'app/api/telemetry/real/route.ts') const telemetryRoutePath = path.join(root, 'app/api/telemetry/real/route.ts')
const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts') const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts')
const vllmApplyRoutePath = path.join(root, 'app/api/inference/vllm/apply/route.ts')
const vllmRestartRoutePath = path.join(root, 'app/api/inference/vllm/restart/route.ts')
const inferenceEndpointUrlPath = path.join(root, 'app/lib/inferenceEndpointUrl.ts') const inferenceEndpointUrlPath = path.join(root, 'app/lib/inferenceEndpointUrl.ts')
const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts') const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts')
const sandboxInferenceRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/route.ts') const sandboxInferenceRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/route.ts')
@@ -33,6 +35,8 @@ const deleteRouteSource = await readFile(deleteRoutePath, 'utf8')
const hostCommandsSource = await readFile(hostCommandsPath, 'utf8') const hostCommandsSource = await readFile(hostCommandsPath, 'utf8')
const telemetryRouteSource = await readFile(telemetryRoutePath, 'utf8') const telemetryRouteSource = await readFile(telemetryRoutePath, 'utf8')
const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8') const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8')
const vllmApplyRouteSource = await readFile(vllmApplyRoutePath, 'utf8')
const vllmRestartRouteSource = await readFile(vllmRestartRoutePath, 'utf8')
const inferenceEndpointUrlSource = await readFile(inferenceEndpointUrlPath, 'utf8') const inferenceEndpointUrlSource = await readFile(inferenceEndpointUrlPath, 'utf8')
const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8') const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8')
const sandboxInferenceRouteSource = await readFile(sandboxInferenceRoutePath, 'utf8') const sandboxInferenceRouteSource = await readFile(sandboxInferenceRoutePath, 'utf8')
@@ -176,11 +180,13 @@ assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--t
assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference route must update existing OpenShell providers') assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference route must update existing OpenShell providers')
assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference route must delete OpenShell providers') assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference route must delete OpenShell providers')
assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route') assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route')
assert.match(inferenceRouteSource, /"vllm"/, 'inference route must accept the dashboard vLLM provider type')
assert.match(inferenceRouteSource, /type === "vllm"[\s\S]*\? "openai" : type/, 'inference route must store vLLM selections as OpenAI-compatible providers')
assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM-compatible endpoints') assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM-compatible endpoints')
assert.match(inferenceRouteSource, /normalizeProviderType\(requestedType, name, baseUrl\)/, 'inference route must force vLLM endpoints through the OpenAI-compatible provider type') assert.match(inferenceRouteSource, /normalizeProviderType\(requestedType, name, baseUrl\)/, 'inference route must force vLLM endpoints through the OpenAI-compatible provider type')
assert.match(inferenceRouteSource, /isOllamaEndpoint\(name, baseUrl\)/, 'inference route must detect local Ollama endpoints') assert.match(inferenceRouteSource, /isOllamaEndpoint\(name, baseUrl\)/, 'inference route must detect local Ollama endpoints')
assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must force local Ollama endpoints through the OpenAI-compatible provider type') assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must force local Ollama endpoints through the OpenAI-compatible provider type')
assert.match(inferenceRouteSource, /body\?\.noVerify !== false \|\| isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must skip verification for local Ollama endpoints') assert.match(inferenceRouteSource, /body\?\.noVerify !== false \|\| requestedType === "vllm" \|\| isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM selections and local Ollama endpoints')
assert.match(inferenceRouteSource, /normalizeInferenceBaseUrlForGateway\(\s*baseUrl,/, 'inference route must normalize endpoint URLs for the OpenShell gateway pod') assert.match(inferenceRouteSource, /normalizeInferenceBaseUrlForGateway\(\s*baseUrl,/, 'inference route must normalize endpoint URLs for the OpenShell gateway pod')
assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{normalizedBaseUrl\}/, 'inference route must pass the normalized endpoint URL as provider config') assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{normalizedBaseUrl\}/, 'inference route must pass the normalized endpoint URL as provider config')
assert.match(inferenceEndpointUrlSource, /host\.docker\.internal/, 'endpoint normalization must rewrite host-local URLs to a gateway-reachable host alias') assert.match(inferenceEndpointUrlSource, /host\.docker\.internal/, 'endpoint normalization must rewrite host-local URLs to a gateway-reachable host alias')
@@ -188,7 +194,35 @@ assert.match(inferenceEndpointUrlSource, /\/chat\/completions/, 'endpoint normal
assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration') assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration')
assert.doesNotMatch(inferencePanelSource, /NVIDIA Preset|useNvidiaInferencePreset|aws\/anthropic\/claude-opus-4-5/, 'settings UI must not expose NVIDIA-corp-specific inference presets') assert.doesNotMatch(inferencePanelSource, /NVIDIA Preset|useNvidiaInferencePreset|aws\/anthropic\/claude-opus-4-5/, 'settings UI must not expose NVIDIA-corp-specific inference presets')
assert.match(inferencePanelSource, /useVllmPreset/, 'settings UI must expose a vLLM endpoint preset') assert.match(inferencePanelSource, /useVllmPreset/, 'settings UI must expose a vLLM endpoint preset')
assert.match(inferencePanelSource, /\{ value: "vllm", label: "vLLM" \}/, 'settings UI must expose vLLM as an explicit provider type')
assert.match(inferencePanelSource, /setType\("vllm"\)/, 'vLLM preset must select the explicit vLLM provider type')
assert.match(inferencePanelSource, /const isVllmEndpointSelected = type === "vllm"/, 'settings UI must only show advanced vLLM controls when the vLLM provider type is selected')
assert.match(inferencePanelSource, /host\.docker\.internal:8000\/v1/, 'vLLM preset must point at the host vLLM OpenAI-compatible endpoint') assert.match(inferencePanelSource, /host\.docker\.internal:8000\/v1/, 'vLLM preset must point at the host vLLM OpenAI-compatible endpoint')
assert.match(inferencePanelSource, /function isVllmEndpoint\(name: string, baseUrl: string\)/, 'settings UI must detect vLLM-compatible endpoint selections')
assert.match(inferencePanelSource, /vLLM Advanced Configuration/, 'settings UI must expose advanced vLLM launch tuning when vLLM is selected')
assert.match(inferencePanelSource, /\/api\/inference\/vllm\/apply/, 'vLLM advanced configuration must apply launch changes through the controller')
assert.match(inferencePanelSource, /\/api\/inference\/vllm\/restart/, 'vLLM advanced configuration must expose a restart action')
assert.match(inferencePanelSource, /--max-model-len/, 'vLLM advanced configuration must generate a max model length launch flag')
assert.match(inferencePanelSource, /--gpu-memory-utilization/, 'vLLM advanced configuration must generate a GPU memory utilization launch flag')
assert.match(inferencePanelSource, /--kv-cache-dtype/, 'vLLM advanced configuration must expose KV cache dtype tuning')
assert.match(inferencePanelSource, /--hf-overrides/, 'vLLM advanced configuration must generate HF overrides for long-context model config')
assert.match(inferencePanelSource, /rope_scaling/, 'vLLM advanced configuration must expose YaRN/RoPE scaling')
assert.match(inferencePanelSource, /max_position_embeddings/, 'vLLM advanced configuration must override max_position_embeddings for long-context kernels')
assert.match(inferencePanelSource, /YaRN\/RoPE scaling/, 'vLLM advanced configuration must let users toggle long-context scaling')
assert.match(inferencePanelSource, /VLLM_ALLOW_LONG_MAX_MODEL_LEN=1/, 'vLLM advanced configuration must include the long max model length safety override for extended contexts')
assert.match(inferencePanelSource, /--enable-auto-tool-choice/, 'vLLM advanced configuration must include the tool-choice launch flag')
assert.match(inferencePanelSource, /--tool-call-parser/, 'vLLM advanced configuration must include the tool-call parser launch flag')
assert.match(vllmApplyRouteSource, /\["rm", "-f", container\]/, 'vLLM apply route must remove the previous container before recreating it')
assert.match(vllmApplyRouteSource, /"run", "-d"/, 'vLLM apply route must launch a replacement vLLM container')
assert.match(vllmApplyRouteSource, /--hf-overrides/, 'vLLM apply route must pass long-context HF overrides to vLLM')
assert.match(vllmApplyRouteSource, /rope_scaling/, 'vLLM apply route must include YaRN/RoPE scaling when enabled')
assert.match(vllmApplyRouteSource, /max_position_embeddings/, 'vLLM apply route must expand max_position_embeddings with max model length')
assert.match(vllmApplyRouteSource, /VLLM_ALLOW_LONG_MAX_MODEL_LEN=1/, 'vLLM apply route must allow vLLM to start above the derived 32K model length')
assert.match(vllmApplyRouteSource, /execFileAsync\(DOCKER_BIN, runArgs/, 'vLLM apply route must execute Docker with argv instead of shell text')
assert.match(vllmRestartRouteSource, /\["restart", container\]/, 'vLLM restart route must restart the selected vLLM container with docker restart')
assert.match(vllmRestartRouteSource, /OPENSHELL_CONTROL_VLLM_CONTAINER/, 'vLLM restart route must support an explicit default container env var')
assert.match(vllmRestartRouteSource, /OPENSHELL_CONTROL_VLLM_CONTAINERS|VLLM_CONTAINER_ALLOWLIST/, 'vLLM restart route must support an explicit container allowlist')
assert.match(vllmRestartRouteSource, /\^vllm/, 'vLLM restart route must restrict generic restarts to vLLM-named containers')
assert.doesNotMatch(inferencePanelSource, /nvidia\/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4/, 'settings UI must not default rebuilt inference routes to the NVIDIA Nemotron model') assert.doesNotMatch(inferencePanelSource, /nvidia\/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4/, 'settings UI must not default rebuilt inference routes to the NVIDIA Nemotron model')
assert.match(inferencePanelSource, /method: "DELETE"/, 'settings UI must expose provider deletion') assert.match(inferencePanelSource, /method: "DELETE"/, 'settings UI must expose provider deletion')
assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models') assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models')