From 05d77aace02b6044a2571fc0b90d995d26877652 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 28 Apr 2026 17:13:59 +0000 Subject: [PATCH] Fix vLLM inference endpoint setup --- app/api/inference/route.ts | 13 ++++++++++-- app/components/InferenceEndpointPanel.tsx | 26 +++++++++++------------ tests/sandbox-lifecycle-check.mjs | 7 ++++-- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/app/api/inference/route.ts b/app/api/inference/route.ts index a89d3cc..bac61d8 100644 --- a/app/api/inference/route.ts +++ b/app/api/inference/route.ts @@ -101,6 +101,14 @@ function optionalString(value: unknown) { return typeof value === "string" ? value.trim() : "" } +function isVllmEndpoint(name: string, baseUrl: string) { + return [name, baseUrl].some((value) => /vllm/i.test(value)) +} + +function normalizeProviderType(type: string, name: string, baseUrl: string) { + return isVllmEndpoint(name, baseUrl) ? "openai" : type +} + function redactSecretOutput(value: string, secrets: string[] = []) { let redacted = value for (const secret of secrets) { @@ -132,14 +140,15 @@ export async function POST(request: Request) { try { const body = await request.json() const name = validateName(body?.name) - const type = validateProviderType(body?.type) + const requestedType = validateProviderType(body?.type) const model = optionalString(body?.model) const baseUrl = optionalString(body?.baseUrl) + const type = normalizeProviderType(requestedType, name, baseUrl) const apiKey = optionalString(body?.apiKey) const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY" const makeActive = body?.makeActive !== false const system = body?.route === "system" || body?.system === true - const noVerify = body?.noVerify !== false + const noVerify = body?.noVerify !== false || isVllmEndpoint(name, baseUrl) const timeout = Number(body?.timeout ?? 0) if (makeActive && !model) throw new Error("model is required when making the endpoint active") diff --git a/app/components/InferenceEndpointPanel.tsx b/app/components/InferenceEndpointPanel.tsx index d4d3b97..9806c2c 100644 --- a/app/components/InferenceEndpointPanel.tsx +++ b/app/components/InferenceEndpointPanel.tsx @@ -36,7 +36,6 @@ type OllamaModel = { const providerTypeOptions = [ { value: "openai", label: "OpenAI compatible" }, - { value: "nvidia", label: "NVIDIA" }, { value: "anthropic", label: "Anthropic" }, { value: "generic", label: "Generic" }, ] @@ -56,11 +55,11 @@ export default function InferenceEndpointPanel() { const [providers, setProviders] = useState([]) const [gateway, setGateway] = useState(emptyRoute) const [system, setSystem] = useState(emptyRoute) - const [name, setName] = useState("nvidia-nim") + const [name, setName] = useState("vllm-local") const [type, setType] = useState("openai") - const [model, setModel] = useState("nvidia/nemotron-3-super-120b-a12b") - const [baseUrl, setBaseUrl] = useState("https://integrate.api.nvidia.com/v1") - const [credentialKey, setCredentialKey] = useState("NVIDIA_API_KEY") + const [model, setModel] = useState("nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4") + const [baseUrl, setBaseUrl] = useState("http://host.docker.internal:8000/v1") + const [credentialKey, setCredentialKey] = useState("OPENAI_API_KEY") const [apiKey, setApiKey] = useState("") const [route, setRoute] = useState<"gateway" | "system">("gateway") const [timeout, setTimeoutValue] = useState("0") @@ -141,15 +140,16 @@ export default function InferenceEndpointPanel() { setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`) } - function useNvidiaInferencePreset() { - setName("nvidia-inference") + function useVllmPreset() { + setName("vllm-local") setType("openai") - setModel("aws/anthropic/claude-opus-4-5") - setBaseUrl("https://inference-api.nvidia.com/v1") - setCredentialKey("NVIDIA_API_KEY") + setModel("nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4") + setBaseUrl("http://host.docker.internal:8000/v1") + setCredentialKey("OPENAI_API_KEY") + setApiKey("") setRoute("gateway") setNoVerify(true) - setMessage("NVIDIA inference preset loaded. Enter an API key, then save the endpoint.") + setMessage("vLLM preset loaded. Adjust the endpoint URL if your server is bound somewhere else, then save.") } async function saveEndpoint() { @@ -333,8 +333,8 @@ export default function InferenceEndpointPanel() { - {message &&

{message}

} diff --git a/tests/sandbox-lifecycle-check.mjs b/tests/sandbox-lifecycle-check.mjs index 761284a..fd0a1ae 100644 --- a/tests/sandbox-lifecycle-check.mjs +++ b/tests/sandbox-lifecycle-check.mjs @@ -75,10 +75,13 @@ assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--t assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference route must update existing OpenShell providers') assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference route must delete OpenShell providers') assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route') +assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM-compatible endpoints') +assert.match(inferenceRouteSource, /normalizeProviderType\(requestedType, name, baseUrl\)/, 'inference route must force vLLM endpoints through the OpenAI-compatible provider type') assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config') assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration') -assert.match(inferencePanelSource, /https:\/\/inference-api\.nvidia\.com\/v1/, 'settings UI must provide the NVIDIA inference API preset') -assert.match(inferencePanelSource, /aws\/anthropic\/claude-opus-4-5/, 'settings UI must include the requested NVIDIA inference model preset') +assert.doesNotMatch(inferencePanelSource, /NVIDIA Preset|useNvidiaInferencePreset|aws\/anthropic\/claude-opus-4-5/, 'settings UI must not expose NVIDIA-corp-specific inference presets') +assert.match(inferencePanelSource, /useVllmPreset/, 'settings UI must expose a vLLM endpoint preset') +assert.match(inferencePanelSource, /host\.docker\.internal:8000\/v1/, 'vLLM preset must point at the host vLLM OpenAI-compatible endpoint') assert.match(inferencePanelSource, /method: "DELETE"/, 'settings UI must expose provider deletion') assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models') assert.match(inferencePanelSource, /window\.setInterval\(\(\) => loadOllamaModels\(\{ silent: true \}\), 10000\)/, 'settings UI must autopoll Ollama while selected')