Add vLLM advanced configuration
This commit is contained in:
@@ -6,7 +6,7 @@ import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
const PROVIDER_TYPES = new Set(["openai", "anthropic", "nvidia", "generic", "claude", "opencode", "codex", "copilot", "gitlab", "github", "outlook"])
|
||||
const PROVIDER_TYPES = new Set(["openai", "vllm", "anthropic", "nvidia", "generic", "claude", "opencode", "codex", "copilot", "gitlab", "github", "outlook"])
|
||||
|
||||
type InferenceRoute = {
|
||||
configured: boolean
|
||||
@@ -111,7 +111,7 @@ function isOllamaEndpoint(name: string, baseUrl: string) {
|
||||
}
|
||||
|
||||
function normalizeProviderType(type: string, name: string, baseUrl: string) {
|
||||
return isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) ? "openai" : type
|
||||
return type === "vllm" || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl) ? "openai" : type
|
||||
}
|
||||
|
||||
function redactSecretOutput(value: string, secrets: string[] = []) {
|
||||
@@ -157,7 +157,7 @@ export async function POST(request: Request) {
|
||||
const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY"
|
||||
const makeActive = body?.makeActive !== false
|
||||
const system = body?.route === "system" || body?.system === true
|
||||
const noVerify = body?.noVerify !== false || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl)
|
||||
const noVerify = body?.noVerify !== false || requestedType === "vllm" || isVllmEndpoint(name, baseUrl) || isOllamaEndpoint(name, baseUrl)
|
||||
const timeout = Number(body?.timeout ?? 0)
|
||||
|
||||
if (makeActive && !model) throw new Error("model is required when making the endpoint active")
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import { NextResponse } from "next/server"
|
||||
import { execFile } from "node:child_process"
|
||||
import { promisify } from "node:util"
|
||||
import { HOST_PATH } from "@/app/lib/hostCommands"
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
const DOCKER_BIN = process.env.DOCKER_BIN || "docker"
|
||||
const DEFAULT_VLLM_CONTAINER = process.env.OPENSHELL_CONTROL_VLLM_CONTAINER || "vllm-bigboy"
|
||||
const DEFAULT_VLLM_IMAGE = "nvcr.io/nvidia/vllm:26.03.post1-py3"
|
||||
const DEFAULT_VLLM_MODEL = "vllm-local"
|
||||
const DEFAULT_VLLM_MAX_MODEL_LEN = "32768"
|
||||
const DEFAULT_VLLM_ORIGINAL_CONTEXT = 32768
|
||||
const DEFAULT_VLLM_YARN_FACTOR = 1
|
||||
|
||||
const DTYPES = new Set(["auto", "bfloat16", "float16", "float", "float32", "half"])
|
||||
const KV_CACHE_DTYPES = new Set(["auto", "bfloat16", "fp8", "fp8_ds_mla", "fp8_e4m3", "fp8_e5m2", "fp8_inc"])
|
||||
const TOOL_CALL_PARSERS = new Set(["", "hermes", "llama3_json", "mistral", "openai", "qwen3_coder"])
|
||||
|
||||
function configuredContainerAllowlist() {
|
||||
return new Set([
|
||||
DEFAULT_VLLM_CONTAINER,
|
||||
...(process.env.OPENSHELL_CONTROL_VLLM_CONTAINERS || process.env.VLLM_CONTAINER_ALLOWLIST || "")
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean),
|
||||
])
|
||||
}
|
||||
|
||||
function text(value: unknown, fallback: string) {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : fallback
|
||||
}
|
||||
|
||||
function bool(value: unknown, fallback: boolean) {
|
||||
return typeof value === "boolean" ? value : fallback
|
||||
}
|
||||
|
||||
function validateContainerName(value: unknown) {
|
||||
const container = text(value, DEFAULT_VLLM_CONTAINER)
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(container)) throw new Error("invalid vLLM container name")
|
||||
const allowlist = configuredContainerAllowlist()
|
||||
if (!allowlist.has(container) && !/^vllm(?:$|[_.-])/i.test(container)) {
|
||||
throw new Error("vLLM container apply is limited to vLLM-named or allowlisted containers")
|
||||
}
|
||||
return container
|
||||
}
|
||||
|
||||
function validateImage(value: unknown) {
|
||||
const image = text(value, DEFAULT_VLLM_IMAGE)
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,255}$/.test(image)) throw new Error("invalid vLLM image")
|
||||
return image
|
||||
}
|
||||
|
||||
function validateModel(value: unknown) {
|
||||
const model = text(value, DEFAULT_VLLM_MODEL)
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9._/-]{0,255}$/.test(model)) throw new Error("invalid vLLM model id")
|
||||
return model
|
||||
}
|
||||
|
||||
function validateGpuDevice(value: unknown) {
|
||||
const gpuDevice = text(value, "0")
|
||||
if (!/^\d+(?:,\d+)*$/.test(gpuDevice)) throw new Error("GPU device must be a comma-separated list of device ids")
|
||||
return gpuDevice
|
||||
}
|
||||
|
||||
function validatePort(value: unknown, fallback: string) {
|
||||
const port = Number.parseInt(text(value, fallback), 10)
|
||||
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("port must be between 1 and 65535")
|
||||
return String(port)
|
||||
}
|
||||
|
||||
function validateInteger(value: unknown, fallback: string, label: string, min: number, max: number) {
|
||||
const parsed = Number.parseInt(text(value, fallback), 10)
|
||||
if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new Error(`${label} must be between ${min} and ${max}`)
|
||||
return String(parsed)
|
||||
}
|
||||
|
||||
function validateOptionalInteger(value: unknown, label: string, min: number, max: number) {
|
||||
const raw = typeof value === "string" ? value.trim() : ""
|
||||
if (!raw) return ""
|
||||
const parsed = Number.parseInt(raw, 10)
|
||||
if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new Error(`${label} must be between ${min} and ${max}`)
|
||||
return String(parsed)
|
||||
}
|
||||
|
||||
function validateMemoryUtilization(value: unknown) {
|
||||
const parsed = Number.parseFloat(text(value, "0.85"))
|
||||
if (!Number.isFinite(parsed) || parsed < 0.1 || parsed > 0.99) throw new Error("GPU memory utilization must be between 0.1 and 0.99")
|
||||
return String(parsed)
|
||||
}
|
||||
|
||||
function validateEnum(value: unknown, fallback: string, allowed: Set<string>, label: string) {
|
||||
const selected = text(value, fallback)
|
||||
if (!allowed.has(selected)) throw new Error(`unsupported ${label}: ${selected}`)
|
||||
return selected
|
||||
}
|
||||
|
||||
function validateYarnFactor(value: unknown) {
|
||||
const parsed = Number.parseFloat(text(value, String(DEFAULT_VLLM_YARN_FACTOR)))
|
||||
if (!Number.isFinite(parsed) || parsed < 1 || parsed > 16) throw new Error("YaRN factor must be between 1 and 16")
|
||||
return parsed
|
||||
}
|
||||
|
||||
function dockerEnv() {
|
||||
return { ...process.env, PATH: HOST_PATH }
|
||||
}
|
||||
|
||||
async function removeExistingContainer(container: string) {
|
||||
try {
|
||||
await execFileAsync(DOCKER_BIN, ["rm", "-f", container], {
|
||||
env: dockerEnv(),
|
||||
timeout: 120000,
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
})
|
||||
} catch (error) {
|
||||
const stderr = String((error as { stderr?: unknown }).stderr || "")
|
||||
if (!/No such container/i.test(stderr)) throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const body = await request.json().catch(() => ({}))
|
||||
const container = validateContainerName(body?.containerName ?? body?.container)
|
||||
const image = validateImage(body?.image)
|
||||
const model = validateModel(body?.model)
|
||||
const gpuDevice = validateGpuDevice(body?.gpuDevice)
|
||||
const hostPort = validatePort(body?.hostPort, "8000")
|
||||
const containerPort = validatePort(body?.containerPort, "8000")
|
||||
const tensorParallelSize = validateInteger(body?.tensorParallelSize, "1", "tensor parallel size", 1, 64)
|
||||
const maxModelLen = validateInteger(body?.maxModelLen, DEFAULT_VLLM_MAX_MODEL_LEN, "max model len", 1024, 262144)
|
||||
const gpuMemoryUtilization = validateMemoryUtilization(body?.gpuMemoryUtilization)
|
||||
const dtype = validateEnum(body?.dtype, "bfloat16", DTYPES, "dtype")
|
||||
const kvCacheDtype = validateEnum(body?.kvCacheDtype, "fp8", KV_CACHE_DTYPES, "KV cache dtype")
|
||||
const maxNumBatchedTokens = validateOptionalInteger(body?.maxNumBatchedTokens, "max batched tokens", 1, 262144)
|
||||
const maxNumSeqs = validateOptionalInteger(body?.maxNumSeqs, "max seqs", 1, 4096)
|
||||
const toolCallParser = validateEnum(body?.toolCallParser, "hermes", TOOL_CALL_PARSERS, "tool parser")
|
||||
const allowLongMaxModelLen = bool(body?.allowLongMaxModelLen, false)
|
||||
const enableYarnScaling = bool(body?.enableYarnScaling, false)
|
||||
const yarnFactor = validateYarnFactor(body?.yarnFactor)
|
||||
const originalContext = Number.parseInt(
|
||||
validateInteger(body?.originalContext, String(DEFAULT_VLLM_ORIGINAL_CONTEXT), "original context", 1024, 262144),
|
||||
10,
|
||||
)
|
||||
const enableChunkedPrefill = bool(body?.enableChunkedPrefill, true)
|
||||
const enableAutoToolChoice = bool(body?.enableAutoToolChoice, true)
|
||||
const calculateKvScales = bool(body?.calculateKvScales, false)
|
||||
const hfCacheDir = `${process.env.HOME || "/root"}/.cache/huggingface`
|
||||
|
||||
const runArgs = [
|
||||
"run", "-d",
|
||||
"--name", container,
|
||||
"--runtime", "nvidia",
|
||||
"--gpus", `"device=${gpuDevice}"`,
|
||||
"--ipc=host",
|
||||
"--shm-size=64g",
|
||||
"--ulimit", "memlock=-1",
|
||||
"--ulimit", "stack=67108864",
|
||||
"-e", `CUDA_VISIBLE_DEVICES=${gpuDevice}`,
|
||||
"-e", "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||
...(allowLongMaxModelLen ? ["-e", "VLLM_ALLOW_LONG_MAX_MODEL_LEN=1"] : []),
|
||||
"-v", `${hfCacheDir}:/root/.cache/huggingface`,
|
||||
"-p", `${hostPort}:${containerPort}`,
|
||||
"--restart", "unless-stopped",
|
||||
image,
|
||||
"python3", "-m", "vllm.entrypoints.openai.api_server",
|
||||
"--model", model,
|
||||
"--host", "0.0.0.0",
|
||||
"--port", containerPort,
|
||||
"--tensor-parallel-size", tensorParallelSize,
|
||||
]
|
||||
|
||||
if (enableYarnScaling) {
|
||||
runArgs.push("--hf-overrides", JSON.stringify({
|
||||
max_position_embeddings: Number.parseInt(maxModelLen, 10),
|
||||
rope_scaling: {
|
||||
type: "yarn",
|
||||
factor: yarnFactor,
|
||||
original_max_position_embeddings: originalContext,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
runArgs.push(
|
||||
"--max-model-len", maxModelLen,
|
||||
"--gpu-memory-utilization", gpuMemoryUtilization,
|
||||
"--dtype", dtype,
|
||||
)
|
||||
|
||||
if (kvCacheDtype !== "auto") runArgs.push("--kv-cache-dtype", kvCacheDtype)
|
||||
if (maxNumBatchedTokens) runArgs.push("--max-num-batched-tokens", maxNumBatchedTokens)
|
||||
if (maxNumSeqs) runArgs.push("--max-num-seqs", maxNumSeqs)
|
||||
if (enableChunkedPrefill) runArgs.push("--enable-chunked-prefill")
|
||||
if (enableAutoToolChoice) {
|
||||
runArgs.push("--enable-auto-tool-choice")
|
||||
if (toolCallParser) runArgs.push("--tool-call-parser", toolCallParser)
|
||||
}
|
||||
if (calculateKvScales && kvCacheDtype.startsWith("fp8")) runArgs.push("--calculate-kv-scales")
|
||||
|
||||
await removeExistingContainer(container)
|
||||
const { stdout, stderr } = await execFileAsync(DOCKER_BIN, runArgs, {
|
||||
env: dockerEnv(),
|
||||
timeout: 120000,
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
ok: true,
|
||||
applied: true,
|
||||
container,
|
||||
image,
|
||||
model,
|
||||
hostPort,
|
||||
containerPort,
|
||||
stdout: String(stdout).trim(),
|
||||
stderr: String(stderr).trim(),
|
||||
note: "vLLM container recreated. The model may take time to load before /v1/models is ready.",
|
||||
})
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Failed to apply vLLM container configuration"
|
||||
return NextResponse.json(
|
||||
{ ok: false, applied: false, error: message },
|
||||
{ status: /invalid|limited|required|must|unsupported|between/.test(message) ? 400 : 500 },
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { NextResponse } from "next/server"
|
||||
import { execFile } from "node:child_process"
|
||||
import { promisify } from "node:util"
|
||||
import { HOST_PATH } from "@/app/lib/hostCommands"
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const DOCKER_BIN = process.env.DOCKER_BIN || "docker"
|
||||
const DEFAULT_VLLM_CONTAINER = process.env.OPENSHELL_CONTROL_VLLM_CONTAINER || "vllm-bigboy"
|
||||
|
||||
function configuredContainerAllowlist() {
|
||||
return new Set([
|
||||
DEFAULT_VLLM_CONTAINER,
|
||||
...(process.env.OPENSHELL_CONTROL_VLLM_CONTAINERS || process.env.VLLM_CONTAINER_ALLOWLIST || "")
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean),
|
||||
])
|
||||
}
|
||||
|
||||
function validateContainerName(value: unknown) {
|
||||
const container = typeof value === "string" && value.trim() ? value.trim() : DEFAULT_VLLM_CONTAINER
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(container)) {
|
||||
throw new Error("invalid vLLM container name")
|
||||
}
|
||||
const allowlist = configuredContainerAllowlist()
|
||||
if (!allowlist.has(container) && !/^vllm(?:$|[_.-])/i.test(container)) {
|
||||
throw new Error("vLLM container restart is limited to vLLM-named or allowlisted containers")
|
||||
}
|
||||
return container
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const body = await request.json().catch(() => ({}))
|
||||
const container = validateContainerName(body?.containerName ?? body?.container)
|
||||
const { stdout, stderr } = await execFileAsync(DOCKER_BIN, ["restart", container], {
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: HOST_PATH,
|
||||
},
|
||||
timeout: 120000,
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
ok: true,
|
||||
restarted: true,
|
||||
container,
|
||||
stdout: String(stdout).trim(),
|
||||
stderr: String(stderr).trim(),
|
||||
note: "vLLM container restart requested. The model may take time to reload before /v1/models is ready.",
|
||||
})
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Failed to restart vLLM container"
|
||||
return NextResponse.json({
|
||||
ok: false,
|
||||
restarted: false,
|
||||
error: message,
|
||||
}, { status: /invalid|limited|required/.test(message) ? 400 : 500 })
|
||||
}
|
||||
}
|
||||
@@ -36,12 +36,33 @@ type OllamaModel = {
|
||||
|
||||
const providerTypeOptions = [
|
||||
{ value: "openai", label: "OpenAI compatible" },
|
||||
{ value: "vllm", label: "vLLM" },
|
||||
{ value: "anthropic", label: "Anthropic" },
|
||||
{ value: "generic", label: "Generic" },
|
||||
]
|
||||
|
||||
const OLLAMA_PROVIDER_NAME = "ollama-local"
|
||||
const OLLAMA_BASE_URL = "http://host.docker.internal:11434/v1"
|
||||
const DEFAULT_VLLM_CONTAINER = "vllm-bigboy"
|
||||
const DEFAULT_VLLM_IMAGE = "nvcr.io/nvidia/vllm:26.03.post1-py3"
|
||||
const DEFAULT_VLLM_MODEL = "vllm-local"
|
||||
const DEFAULT_VLLM_MAX_MODEL_LEN = "32768"
|
||||
const DEFAULT_VLLM_YARN_FACTOR = "1.0"
|
||||
const DEFAULT_VLLM_ORIGINAL_CONTEXT = "32768"
|
||||
|
||||
function isVllmEndpoint(name: string, baseUrl: string) {
|
||||
return [name, baseUrl].some((value) => /vllm/i.test(value))
|
||||
}
|
||||
|
||||
function shellQuote(value: string) {
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed) return "''"
|
||||
return /^[A-Za-z0-9_./:@%+=,-]+$/.test(trimmed) ? trimmed : `'${trimmed.replace(/'/g, "'\\''")}'`
|
||||
}
|
||||
|
||||
function commandLine(parts: string[]) {
|
||||
return parts.filter(Boolean).join(" \\\n ")
|
||||
}
|
||||
|
||||
function emptyRoute(): InferenceRoute {
|
||||
return { configured: false, provider: null, model: null, version: null, timeout: null }
|
||||
@@ -51,6 +72,324 @@ function FieldLabel({ children }: { children: React.ReactNode }) {
|
||||
return <label className="text-[10px] uppercase tracking-wider text-[var(--foreground-dim)]">{children}</label>
|
||||
}
|
||||
|
||||
function VllmAdvancedConfiguration({
|
||||
model,
|
||||
port,
|
||||
}: {
|
||||
model: string
|
||||
port: string
|
||||
}) {
|
||||
const [containerName, setContainerName] = useState(DEFAULT_VLLM_CONTAINER)
|
||||
const [image, setImage] = useState(DEFAULT_VLLM_IMAGE)
|
||||
const [gpuDevice, setGpuDevice] = useState("0")
|
||||
const [hostPort, setHostPort] = useState("8000")
|
||||
const [containerPort, setContainerPort] = useState("8000")
|
||||
const [tensorParallelSize, setTensorParallelSize] = useState("1")
|
||||
const [maxModelLen, setMaxModelLen] = useState(DEFAULT_VLLM_MAX_MODEL_LEN)
|
||||
const [gpuMemoryUtilization, setGpuMemoryUtilization] = useState("0.85")
|
||||
const [dtype, setDtype] = useState("bfloat16")
|
||||
const [kvCacheDtype, setKvCacheDtype] = useState("fp8")
|
||||
const [maxNumBatchedTokens, setMaxNumBatchedTokens] = useState("8192")
|
||||
const [maxNumSeqs, setMaxNumSeqs] = useState("")
|
||||
const [toolCallParser, setToolCallParser] = useState("hermes")
|
||||
const [allowLongMaxModelLen, setAllowLongMaxModelLen] = useState(false)
|
||||
const [enableYarnScaling, setEnableYarnScaling] = useState(false)
|
||||
const [yarnFactor, setYarnFactor] = useState(DEFAULT_VLLM_YARN_FACTOR)
|
||||
const [originalContext, setOriginalContext] = useState(DEFAULT_VLLM_ORIGINAL_CONTEXT)
|
||||
const [enableChunkedPrefill, setEnableChunkedPrefill] = useState(true)
|
||||
const [enableAutoToolChoice, setEnableAutoToolChoice] = useState(true)
|
||||
const [calculateKvScales, setCalculateKvScales] = useState(false)
|
||||
const [copied, setCopied] = useState(false)
|
||||
const [applying, setApplying] = useState(false)
|
||||
const [restarting, setRestarting] = useState(false)
|
||||
const [restartMessage, setRestartMessage] = useState("")
|
||||
|
||||
useEffect(() => {
|
||||
if (port && hostPort === "8000") setHostPort(port)
|
||||
if (port && containerPort === "8000") setContainerPort(port)
|
||||
}, [containerPort, hostPort, port])
|
||||
|
||||
const maxInputWithDefaultOutput = useMemo(() => {
|
||||
const parsed = Number.parseInt(maxModelLen, 10)
|
||||
return Number.isFinite(parsed) ? Math.max(0, parsed - 4096) : 0
|
||||
}, [maxModelLen])
|
||||
|
||||
const hfOverridesJson = useMemo(() => {
|
||||
const factor = Number.parseFloat(yarnFactor)
|
||||
const originalMaxPositionEmbeddings = Number.parseInt(originalContext, 10)
|
||||
const maxPositionEmbeddings = Number.parseInt(maxModelLen, 10)
|
||||
return JSON.stringify({
|
||||
max_position_embeddings: Number.isFinite(maxPositionEmbeddings) && maxPositionEmbeddings > 0
|
||||
? maxPositionEmbeddings
|
||||
: Number.parseInt(DEFAULT_VLLM_MAX_MODEL_LEN, 10),
|
||||
rope_scaling: {
|
||||
type: "yarn",
|
||||
factor: Number.isFinite(factor) && factor > 0 ? factor : Number.parseFloat(DEFAULT_VLLM_YARN_FACTOR),
|
||||
original_max_position_embeddings: Number.isFinite(originalMaxPositionEmbeddings) && originalMaxPositionEmbeddings > 0
|
||||
? originalMaxPositionEmbeddings
|
||||
: Number.parseInt(DEFAULT_VLLM_ORIGINAL_CONTEXT, 10),
|
||||
},
|
||||
})
|
||||
}, [maxModelLen, originalContext, yarnFactor])
|
||||
|
||||
const runCommand = useMemo(() => {
|
||||
const dockerArgs = [
|
||||
"docker run -d",
|
||||
`--name ${shellQuote(containerName || DEFAULT_VLLM_CONTAINER)}`,
|
||||
"--runtime nvidia",
|
||||
`--gpus ${shellQuote(`"device=${gpuDevice || "0"}"`)}`,
|
||||
"--ipc=host",
|
||||
"--shm-size=64g",
|
||||
"--ulimit memlock=-1",
|
||||
"--ulimit stack=67108864",
|
||||
`-e CUDA_VISIBLE_DEVICES=${shellQuote(gpuDevice || "0")}`,
|
||||
"-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||
allowLongMaxModelLen ? "-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1" : "",
|
||||
`-v "$HOME/.cache/huggingface:/root/.cache/huggingface"`,
|
||||
`-p ${shellQuote(hostPort || "8000")}:${shellQuote(containerPort || "8000")}`,
|
||||
"--restart unless-stopped",
|
||||
shellQuote(image || DEFAULT_VLLM_IMAGE),
|
||||
"python3 -m vllm.entrypoints.openai.api_server",
|
||||
`--model ${shellQuote(model || DEFAULT_VLLM_MODEL)}`,
|
||||
"--host 0.0.0.0",
|
||||
`--port ${shellQuote(containerPort || "8000")}`,
|
||||
`--tensor-parallel-size ${shellQuote(tensorParallelSize || "1")}`,
|
||||
enableYarnScaling ? `--hf-overrides ${shellQuote(hfOverridesJson)}` : "",
|
||||
`--max-model-len ${shellQuote(maxModelLen || DEFAULT_VLLM_MAX_MODEL_LEN)}`,
|
||||
`--gpu-memory-utilization ${shellQuote(gpuMemoryUtilization || "0.85")}`,
|
||||
`--dtype ${shellQuote(dtype || "bfloat16")}`,
|
||||
kvCacheDtype && kvCacheDtype !== "auto" ? `--kv-cache-dtype ${shellQuote(kvCacheDtype)}` : "",
|
||||
maxNumBatchedTokens ? `--max-num-batched-tokens ${shellQuote(maxNumBatchedTokens)}` : "",
|
||||
maxNumSeqs ? `--max-num-seqs ${shellQuote(maxNumSeqs)}` : "",
|
||||
enableChunkedPrefill ? "--enable-chunked-prefill" : "",
|
||||
enableAutoToolChoice ? "--enable-auto-tool-choice" : "",
|
||||
enableAutoToolChoice && toolCallParser ? `--tool-call-parser ${shellQuote(toolCallParser)}` : "",
|
||||
calculateKvScales && kvCacheDtype.startsWith("fp8") ? "--calculate-kv-scales" : "",
|
||||
]
|
||||
return commandLine(dockerArgs)
|
||||
}, [
|
||||
allowLongMaxModelLen,
|
||||
calculateKvScales,
|
||||
containerName,
|
||||
containerPort,
|
||||
dtype,
|
||||
enableAutoToolChoice,
|
||||
enableChunkedPrefill,
|
||||
enableYarnScaling,
|
||||
gpuDevice,
|
||||
gpuMemoryUtilization,
|
||||
hfOverridesJson,
|
||||
hostPort,
|
||||
image,
|
||||
kvCacheDtype,
|
||||
maxModelLen,
|
||||
maxNumBatchedTokens,
|
||||
maxNumSeqs,
|
||||
model,
|
||||
tensorParallelSize,
|
||||
toolCallParser,
|
||||
])
|
||||
|
||||
async function copyRunCommand() {
|
||||
try {
|
||||
await navigator.clipboard.writeText(runCommand)
|
||||
setCopied(true)
|
||||
window.setTimeout(() => setCopied(false), 1600)
|
||||
} catch {
|
||||
setCopied(false)
|
||||
}
|
||||
}
|
||||
|
||||
async function applyVllmContainer() {
|
||||
const target = containerName.trim() || DEFAULT_VLLM_CONTAINER
|
||||
if (!window.confirm(`Recreate vLLM container '${target}' with this configuration?`)) return
|
||||
try {
|
||||
setApplying(true)
|
||||
setRestartMessage("")
|
||||
const response = await fetch("/api/inference/vllm/apply", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
containerName: target,
|
||||
image,
|
||||
gpuDevice,
|
||||
hostPort,
|
||||
containerPort,
|
||||
model: model || DEFAULT_VLLM_MODEL,
|
||||
tensorParallelSize,
|
||||
maxModelLen,
|
||||
gpuMemoryUtilization,
|
||||
dtype,
|
||||
kvCacheDtype,
|
||||
maxNumBatchedTokens,
|
||||
maxNumSeqs,
|
||||
toolCallParser,
|
||||
allowLongMaxModelLen,
|
||||
enableYarnScaling,
|
||||
yarnFactor,
|
||||
originalContext,
|
||||
enableChunkedPrefill,
|
||||
enableAutoToolChoice,
|
||||
calculateKvScales,
|
||||
}),
|
||||
})
|
||||
const data = await response.json()
|
||||
if (!response.ok) throw new Error(data.error || "Failed to apply vLLM configuration")
|
||||
setRestartMessage(`Recreated ${data.container || target}. Wait for the model to finish loading before sending requests.`)
|
||||
} catch (error) {
|
||||
setRestartMessage(error instanceof Error ? error.message : "Failed to apply vLLM configuration")
|
||||
} finally {
|
||||
setApplying(false)
|
||||
}
|
||||
}
|
||||
|
||||
async function restartVllmContainer() {
|
||||
const target = containerName.trim() || DEFAULT_VLLM_CONTAINER
|
||||
if (!window.confirm(`Restart vLLM container '${target}'?`)) return
|
||||
try {
|
||||
setRestarting(true)
|
||||
setRestartMessage("")
|
||||
const response = await fetch("/api/inference/vllm/restart", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ containerName: target }),
|
||||
})
|
||||
const data = await response.json()
|
||||
if (!response.ok) throw new Error(data.error || "Failed to restart vLLM container")
|
||||
setRestartMessage(`Restarted ${data.container || target}. Wait for the model to finish loading before sending requests.`)
|
||||
} catch (error) {
|
||||
setRestartMessage(error instanceof Error ? error.message : "Failed to restart vLLM container")
|
||||
} finally {
|
||||
setRestarting(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] p-4">
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div>
|
||||
<h3 className="text-xs uppercase tracking-wider text-[var(--foreground)]">vLLM Advanced Configuration</h3>
|
||||
<p className="mt-1 text-xs text-[var(--foreground-dim)]">Input budget at 4096 output tokens: {maxInputWithDefaultOutput.toLocaleString()} tokens.</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<button type="button" onClick={applyVllmContainer} disabled={applying || restarting} className="px-3 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
|
||||
{applying ? "Applying" : "Apply"}
|
||||
</button>
|
||||
<button type="button" onClick={restartVllmContainer} disabled={restarting || applying} className="px-3 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
|
||||
{restarting ? "Restarting" : "Restart"}
|
||||
</button>
|
||||
<button type="button" onClick={copyRunCommand} className="px-3 py-2 rounded-sm bg-[var(--background)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:bg-[var(--background-panel)]">
|
||||
{copied ? "Copied" : "Copy"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Container</FieldLabel>
|
||||
<input value={containerName} onChange={(event) => setContainerName(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2 md:col-span-2">
|
||||
<FieldLabel>Image</FieldLabel>
|
||||
<input value={image} onChange={(event) => setImage(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>GPU Device</FieldLabel>
|
||||
<input value={gpuDevice} onChange={(event) => setGpuDevice(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Host Port</FieldLabel>
|
||||
<input value={hostPort} onChange={(event) => setHostPort(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Container Port</FieldLabel>
|
||||
<input value={containerPort} onChange={(event) => setContainerPort(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Max Model Len</FieldLabel>
|
||||
<input value={maxModelLen} onChange={(event) => setMaxModelLen(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>GPU Memory</FieldLabel>
|
||||
<input value={gpuMemoryUtilization} onChange={(event) => setGpuMemoryUtilization(event.target.value)} inputMode="decimal" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Tensor Parallel</FieldLabel>
|
||||
<input value={tensorParallelSize} onChange={(event) => setTensorParallelSize(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>dtype</FieldLabel>
|
||||
<select value={dtype} onChange={(event) => setDtype(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]">
|
||||
<option value="bfloat16">bfloat16</option>
|
||||
<option value="float16">float16</option>
|
||||
<option value="auto">auto</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>KV Cache dtype</FieldLabel>
|
||||
<select value={kvCacheDtype} onChange={(event) => setKvCacheDtype(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]">
|
||||
<option value="fp8">fp8</option>
|
||||
<option value="fp8_e4m3">fp8_e4m3</option>
|
||||
<option value="fp8_e5m2">fp8_e5m2</option>
|
||||
<option value="auto">auto</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Tool Parser</FieldLabel>
|
||||
<select value={toolCallParser} onChange={(event) => setToolCallParser(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]">
|
||||
<option value="hermes">hermes</option>
|
||||
<option value="llama3_json">llama3_json</option>
|
||||
<option value="mistral">mistral</option>
|
||||
<option value="">none</option>
|
||||
</select>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Max Batched Tokens</FieldLabel>
|
||||
<input value={maxNumBatchedTokens} onChange={(event) => setMaxNumBatchedTokens(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Max Seqs</FieldLabel>
|
||||
<input value={maxNumSeqs} onChange={(event) => setMaxNumSeqs(event.target.value)} inputMode="numeric" placeholder="auto" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>YaRN Factor</FieldLabel>
|
||||
<input value={yarnFactor} onChange={(event) => setYarnFactor(event.target.value)} inputMode="decimal" disabled={!enableYarnScaling} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)] disabled:opacity-50" />
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<FieldLabel>Original Context</FieldLabel>
|
||||
<input value={originalContext} onChange={(event) => setOriginalContext(event.target.value)} inputMode="numeric" disabled={!enableYarnScaling} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)] disabled:opacity-50" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 grid grid-cols-1 md:grid-cols-3 gap-3">
|
||||
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
|
||||
<input type="checkbox" checked={enableYarnScaling} onChange={(event) => setEnableYarnScaling(event.target.checked)} />
|
||||
YaRN/RoPE scaling
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
|
||||
<input type="checkbox" checked={allowLongMaxModelLen} onChange={(event) => setAllowLongMaxModelLen(event.target.checked)} />
|
||||
Allow long max len
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
|
||||
<input type="checkbox" checked={enableChunkedPrefill} onChange={(event) => setEnableChunkedPrefill(event.target.checked)} />
|
||||
Chunked prefill
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
|
||||
<input type="checkbox" checked={enableAutoToolChoice} onChange={(event) => setEnableAutoToolChoice(event.target.checked)} />
|
||||
Auto tool choice
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-xs text-[var(--foreground)]">
|
||||
<input type="checkbox" checked={calculateKvScales} onChange={(event) => setCalculateKvScales(event.target.checked)} />
|
||||
Calculate KV scales
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<textarea readOnly value={runCommand} className="mt-4 min-h-[240px] w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] p-3 text-[11px] font-mono leading-5 text-[var(--foreground)] focus:outline-none" />
|
||||
{restartMessage && <p className="mt-3 text-xs text-[var(--foreground-dim)]">{restartMessage}</p>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function InferenceEndpointPanel() {
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [saving, setSaving] = useState(false)
|
||||
@@ -77,6 +416,14 @@ export default function InferenceEndpointPanel() {
|
||||
[providers, gateway.provider]
|
||||
)
|
||||
const isOllamaEndpoint = name.toLowerCase().includes("ollama") || baseUrl.includes("11434")
|
||||
const isVllmEndpointSelected = type === "vllm"
|
||||
const vllmPort = useMemo(() => {
|
||||
try {
|
||||
return new URL(baseUrl).port || "8000"
|
||||
} catch {
|
||||
return "8000"
|
||||
}
|
||||
}, [baseUrl])
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
@@ -90,6 +437,7 @@ export default function InferenceEndpointPanel() {
|
||||
setProviders(Array.isArray(data.providers) ? data.providers : [])
|
||||
if (data.gateway?.provider) setName(data.gateway.provider)
|
||||
if (data.gateway?.model) setModel(data.gateway.model)
|
||||
if (data.gateway?.provider && isVllmEndpoint(data.gateway.provider, "")) setType("vllm")
|
||||
} catch (error) {
|
||||
setMessage(error instanceof Error ? error.message : "Failed to load inference configuration")
|
||||
} finally {
|
||||
@@ -145,7 +493,7 @@ export default function InferenceEndpointPanel() {
|
||||
function selectProvider(provider: ProviderSummary) {
|
||||
if (!provider.name) return
|
||||
setName(provider.name)
|
||||
setType(provider.type || "openai")
|
||||
setType(isVllmEndpoint(provider.name, "") ? "vllm" : provider.type || "openai")
|
||||
setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY")
|
||||
if (provider.name.toLowerCase().includes("ollama")) {
|
||||
setBaseUrl(OLLAMA_BASE_URL)
|
||||
@@ -156,8 +504,8 @@ export default function InferenceEndpointPanel() {
|
||||
|
||||
function useVllmPreset() {
|
||||
setName("vllm-local")
|
||||
setType("openai")
|
||||
setModel("vllm-local")
|
||||
setType("vllm")
|
||||
setModel(DEFAULT_VLLM_MODEL)
|
||||
setBaseUrl("http://host.docker.internal:8000/v1")
|
||||
setCredentialKey("OPENAI_API_KEY")
|
||||
setApiKey("")
|
||||
@@ -343,6 +691,10 @@ export default function InferenceEndpointPanel() {
|
||||
Skip provider verification while saving
|
||||
</label>
|
||||
|
||||
{isVllmEndpointSelected && (
|
||||
<VllmAdvancedConfiguration model={model} port={vllmPort} />
|
||||
)}
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
<button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
|
||||
{saving ? "Saving..." : "Save Endpoint"}
|
||||
|
||||
Reference in New Issue
Block a user