Fix vLLM inference endpoint setup

This commit is contained in:
Codex
2026-04-28 17:14:09 +00:00
parent 476d3a96bb
commit 05d77aace0
3 changed files with 29 additions and 17 deletions
+11 -2
View File
@@ -101,6 +101,14 @@ function optionalString(value: unknown) {
return typeof value === "string" ? value.trim() : "" return typeof value === "string" ? value.trim() : ""
} }
function isVllmEndpoint(name: string, baseUrl: string) {
return [name, baseUrl].some((value) => /vllm/i.test(value))
}
function normalizeProviderType(type: string, name: string, baseUrl: string) {
return isVllmEndpoint(name, baseUrl) ? "openai" : type
}
function redactSecretOutput(value: string, secrets: string[] = []) { function redactSecretOutput(value: string, secrets: string[] = []) {
let redacted = value let redacted = value
for (const secret of secrets) { for (const secret of secrets) {
@@ -132,14 +140,15 @@ export async function POST(request: Request) {
try { try {
const body = await request.json() const body = await request.json()
const name = validateName(body?.name) const name = validateName(body?.name)
const type = validateProviderType(body?.type) const requestedType = validateProviderType(body?.type)
const model = optionalString(body?.model) const model = optionalString(body?.model)
const baseUrl = optionalString(body?.baseUrl) const baseUrl = optionalString(body?.baseUrl)
const type = normalizeProviderType(requestedType, name, baseUrl)
const apiKey = optionalString(body?.apiKey) const apiKey = optionalString(body?.apiKey)
const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY" const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY"
const makeActive = body?.makeActive !== false const makeActive = body?.makeActive !== false
const system = body?.route === "system" || body?.system === true const system = body?.route === "system" || body?.system === true
const noVerify = body?.noVerify !== false const noVerify = body?.noVerify !== false || isVllmEndpoint(name, baseUrl)
const timeout = Number(body?.timeout ?? 0) const timeout = Number(body?.timeout ?? 0)
if (makeActive && !model) throw new Error("model is required when making the endpoint active") if (makeActive && !model) throw new Error("model is required when making the endpoint active")
+13 -13
View File
@@ -36,7 +36,6 @@ type OllamaModel = {
const providerTypeOptions = [ const providerTypeOptions = [
{ value: "openai", label: "OpenAI compatible" }, { value: "openai", label: "OpenAI compatible" },
{ value: "nvidia", label: "NVIDIA" },
{ value: "anthropic", label: "Anthropic" }, { value: "anthropic", label: "Anthropic" },
{ value: "generic", label: "Generic" }, { value: "generic", label: "Generic" },
] ]
@@ -56,11 +55,11 @@ export default function InferenceEndpointPanel() {
const [providers, setProviders] = useState<ProviderSummary[]>([]) const [providers, setProviders] = useState<ProviderSummary[]>([])
const [gateway, setGateway] = useState<InferenceRoute>(emptyRoute) const [gateway, setGateway] = useState<InferenceRoute>(emptyRoute)
const [system, setSystem] = useState<InferenceRoute>(emptyRoute) const [system, setSystem] = useState<InferenceRoute>(emptyRoute)
const [name, setName] = useState("nvidia-nim") const [name, setName] = useState("vllm-local")
const [type, setType] = useState("openai") const [type, setType] = useState("openai")
const [model, setModel] = useState("nvidia/nemotron-3-super-120b-a12b") const [model, setModel] = useState("nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4")
const [baseUrl, setBaseUrl] = useState("https://integrate.api.nvidia.com/v1") const [baseUrl, setBaseUrl] = useState("http://host.docker.internal:8000/v1")
const [credentialKey, setCredentialKey] = useState("NVIDIA_API_KEY") const [credentialKey, setCredentialKey] = useState("OPENAI_API_KEY")
const [apiKey, setApiKey] = useState("") const [apiKey, setApiKey] = useState("")
const [route, setRoute] = useState<"gateway" | "system">("gateway") const [route, setRoute] = useState<"gateway" | "system">("gateway")
const [timeout, setTimeoutValue] = useState("0") const [timeout, setTimeoutValue] = useState("0")
@@ -141,15 +140,16 @@ export default function InferenceEndpointPanel() {
setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`) setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`)
} }
function useNvidiaInferencePreset() { function useVllmPreset() {
setName("nvidia-inference") setName("vllm-local")
setType("openai") setType("openai")
setModel("aws/anthropic/claude-opus-4-5") setModel("nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4")
setBaseUrl("https://inference-api.nvidia.com/v1") setBaseUrl("http://host.docker.internal:8000/v1")
setCredentialKey("NVIDIA_API_KEY") setCredentialKey("OPENAI_API_KEY")
setApiKey("")
setRoute("gateway") setRoute("gateway")
setNoVerify(true) setNoVerify(true)
setMessage("NVIDIA inference preset loaded. Enter an API key, then save the endpoint.") setMessage("vLLM preset loaded. Adjust the endpoint URL if your server is bound somewhere else, then save.")
} }
async function saveEndpoint() { async function saveEndpoint() {
@@ -333,8 +333,8 @@ export default function InferenceEndpointPanel() {
<button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50"> <button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
{saving ? "Saving..." : "Save Endpoint"} {saving ? "Saving..." : "Save Endpoint"}
</button> </button>
<button type="button" onClick={useNvidiaInferencePreset} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--background-tertiary)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:bg-[var(--background-secondary)] disabled:opacity-50"> <button type="button" onClick={useVllmPreset} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--background-tertiary)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:bg-[var(--background-secondary)] disabled:opacity-50">
NVIDIA Preset vLLM Preset
</button> </button>
{message && <p className="text-xs text-[var(--foreground-dim)] whitespace-pre-wrap">{message}</p>} {message && <p className="text-xs text-[var(--foreground-dim)] whitespace-pre-wrap">{message}</p>}
</div> </div>
+5 -2
View File
@@ -75,10 +75,13 @@ assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--t
assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference route must update existing OpenShell providers') assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference route must update existing OpenShell providers')
assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference route must delete OpenShell providers') assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference route must delete OpenShell providers')
assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route') assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route')
assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM-compatible endpoints')
assert.match(inferenceRouteSource, /normalizeProviderType\(requestedType, name, baseUrl\)/, 'inference route must force vLLM endpoints through the OpenAI-compatible provider type')
assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config') assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config')
assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration') assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration')
assert.match(inferencePanelSource, /https:\/\/inference-api\.nvidia\.com\/v1/, 'settings UI must provide the NVIDIA inference API preset') assert.doesNotMatch(inferencePanelSource, /NVIDIA Preset|useNvidiaInferencePreset|aws\/anthropic\/claude-opus-4-5/, 'settings UI must not expose NVIDIA-corp-specific inference presets')
assert.match(inferencePanelSource, /aws\/anthropic\/claude-opus-4-5/, 'settings UI must include the requested NVIDIA inference model preset') assert.match(inferencePanelSource, /useVllmPreset/, 'settings UI must expose a vLLM endpoint preset')
assert.match(inferencePanelSource, /host\.docker\.internal:8000\/v1/, 'vLLM preset must point at the host vLLM OpenAI-compatible endpoint')
assert.match(inferencePanelSource, /method: "DELETE"/, 'settings UI must expose provider deletion') assert.match(inferencePanelSource, /method: "DELETE"/, 'settings UI must expose provider deletion')
assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models') assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models')
assert.match(inferencePanelSource, /window\.setInterval\(\(\) => loadOllamaModels\(\{ silent: true \}\), 10000\)/, 'settings UI must autopoll Ollama while selected') assert.match(inferencePanelSource, /window\.setInterval\(\(\) => loadOllamaModels\(\{ silent: true \}\), 10000\)/, 'settings UI must autopoll Ollama while selected')