From e55dc7b9e07f7b488312377c6e82b1443c403904 Mon Sep 17 00:00:00 2001 From: mmckeen-nv Date: Wed, 29 Apr 2026 13:01:47 -0700 Subject: [PATCH] Normalize inference endpoint URLs --- app/api/inference/route.ts | 9 +++++-- app/api/sandbox/create/route.ts | 4 ++- app/lib/inferenceEndpointUrl.ts | 42 +++++++++++++++++++++++++++++++ app/lib/sandboxInferenceApply.ts | 6 ++--- tests/sandbox-lifecycle-check.mjs | 9 ++++++- 5 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 app/lib/inferenceEndpointUrl.ts diff --git a/app/api/inference/route.ts b/app/api/inference/route.ts index bac61d8..d886b75 100644 --- a/app/api/inference/route.ts +++ b/app/api/inference/route.ts @@ -1,6 +1,7 @@ import { NextResponse } from "next/server" import { execFile } from "node:child_process" import { promisify } from "node:util" +import { normalizeInferenceBaseUrlForGateway } from "@/app/lib/inferenceEndpointUrl" import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands" const execFileAsync = promisify(execFile) @@ -144,6 +145,10 @@ export async function POST(request: Request) { const model = optionalString(body?.model) const baseUrl = optionalString(body?.baseUrl) const type = normalizeProviderType(requestedType, name, baseUrl) + const normalizedBaseUrl = normalizeInferenceBaseUrlForGateway( + baseUrl, + type === "anthropic" ? "anthropic" : "openai", + ) const apiKey = optionalString(body?.apiKey) const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY" const makeActive = body?.makeActive !== false @@ -153,7 +158,7 @@ export async function POST(request: Request) { if (makeActive && !model) throw new Error("model is required when making the endpoint active") if (!/^[A-Z_][A-Z0-9_]*$/.test(credentialKey)) throw new Error("credential key must look like an environment variable name") - if (baseUrl) new URL(baseUrl) + if (normalizedBaseUrl) new URL(normalizedBaseUrl) const existingNames = (await readProviders()).map((provider) => provider.name).filter(Boolean) const providerArgs = existingNames.includes(name) @@ -163,7 +168,7 @@ export async function POST(request: Request) { const providerEnv = apiKey ? { [credentialKey]: apiKey } : {} if (!existingNames.includes(name) && !apiKey) providerArgs.push("--from-existing") if (apiKey) providerArgs.push("--credential", credentialKey) - if (baseUrl) providerArgs.push("--config", `OPENAI_BASE_URL=${baseUrl}`) + if (normalizedBaseUrl) providerArgs.push("--config", `OPENAI_BASE_URL=${normalizedBaseUrl}`) const providerResult = await runOpenShell(providerArgs, providerEnv) let inferenceResult = null diff --git a/app/api/sandbox/create/route.ts b/app/api/sandbox/create/route.ts index bd29238..7f6af5c 100644 --- a/app/api/sandbox/create/route.ts +++ b/app/api/sandbox/create/route.ts @@ -4,6 +4,7 @@ import { existsSync } from "node:fs" import { readFile, writeFile } from "node:fs/promises" import path from "node:path" import { promisify } from "node:util" +import { normalizeInferenceBaseUrlForGateway } from "@/app/lib/inferenceEndpointUrl" import { inspectSandbox } from "@/app/lib/openshellHost" import { commandExists, @@ -127,7 +128,8 @@ function applyConfiguredInferenceToOnboardEnv(env: NodeJS.ProcessEnv, route: Con } if (provider === "custom") { const endpointUrl = process.env.NEMOCLAW_ENDPOINT_URL || process.env.OPENAI_BASE_URL || process.env.VLLM_BASE_URL || "" - if (endpointUrl) env.NEMOCLAW_ENDPOINT_URL = endpointUrl + const normalizedEndpointUrl = normalizeInferenceBaseUrlForGateway(endpointUrl, "openai") + if (normalizedEndpointUrl) env.NEMOCLAW_ENDPOINT_URL = normalizedEndpointUrl const providerKey = process.env.NEMOCLAW_PROVIDER_KEY || process.env.OPENAI_API_KEY || process.env.VLLM_API_KEY || "" if (providerKey) env.NEMOCLAW_PROVIDER_KEY = providerKey } diff --git a/app/lib/inferenceEndpointUrl.ts b/app/lib/inferenceEndpointUrl.ts new file mode 100644 index 0000000..1cbe126 --- /dev/null +++ b/app/lib/inferenceEndpointUrl.ts @@ -0,0 +1,42 @@ +export type InferenceEndpointFlavor = "anthropic" | "openai" + +const HOST_REACHABLE_FROM_OPENSHELL = "host.docker.internal" +const LOCAL_GATEWAY_HOSTS = new Set(["localhost", "127.0.0.1", "0.0.0.0", "::1"]) + +function stripEndpointSuffix(pathname = "", suffixes: string[] = []) { + for (const suffix of suffixes) { + if (pathname === suffix) return "" + if (pathname.endsWith(suffix)) return pathname.slice(0, -suffix.length) + } + return pathname +} + +function endpointSuffixes(flavor: InferenceEndpointFlavor) { + return flavor === "anthropic" + ? ["/v1/messages", "/v1/models", "/v1", "/messages", "/models"] + : ["/responses", "/chat/completions", "/completions", "/models"] +} + +function ensureScheme(value: string) { + return /^[a-z][a-z0-9+.-]*:\/\//i.test(value) ? value : `http://${value}` +} + +export function normalizeInferenceBaseUrlForGateway(value: string, flavor: InferenceEndpointFlavor = "openai") { + const raw = String(value || "").trim() + if (!raw) return "" + + try { + const url = new URL(ensureScheme(raw)) + url.search = "" + url.hash = "" + if (LOCAL_GATEWAY_HOSTS.has(url.hostname.toLowerCase())) { + url.hostname = HOST_REACHABLE_FROM_OPENSHELL + } + let pathname = stripEndpointSuffix(url.pathname.replace(/\/+$/, ""), endpointSuffixes(flavor)) + pathname = pathname.replace(/\/+$/, "") + url.pathname = pathname || "/" + return url.pathname === "/" ? url.origin : `${url.origin}${url.pathname}` + } catch { + return raw.replace(/[?#].*$/, "").replace(/\/+$/, "") + } +} diff --git a/app/lib/sandboxInferenceApply.ts b/app/lib/sandboxInferenceApply.ts index 8a20eaf..41fd368 100644 --- a/app/lib/sandboxInferenceApply.ts +++ b/app/lib/sandboxInferenceApply.ts @@ -7,11 +7,11 @@ const execFileAsync = promisify(execFile) const DOCKER_BIN = process.env.DOCKER_BIN || "docker" const OPENSHELL_CLUSTER_CONTAINER = process.env.OPENSHELL_CLUSTER_CONTAINER || "openshell-cluster-nemoclaw" -function modelEntry(route: SandboxInferenceRoute, openClawModelRef: string, compat: Record | null) { +function modelEntry(route: SandboxInferenceRoute, compat: Record | null) { return { ...(compat ? { compat } : {}), id: route.model, - name: openClawModelRef, + name: route.model, reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, @@ -78,7 +78,7 @@ function buildOpenClawConfig(current: any, routes: SandboxInferenceRoute[], prim api: resolved.api, models: [], } - providers[resolved.providerKey].models.push(modelEntry(route, resolved.modelRef, resolved.compat)) + providers[resolved.providerKey].models.push(modelEntry(route, resolved.compat)) if (route.id === primary.id) primaryModelRef = resolved.modelRef } diff --git a/tests/sandbox-lifecycle-check.mjs b/tests/sandbox-lifecycle-check.mjs index b0526f6..8e5eaef 100644 --- a/tests/sandbox-lifecycle-check.mjs +++ b/tests/sandbox-lifecycle-check.mjs @@ -8,6 +8,7 @@ const deleteRoutePath = path.join(root, 'app/api/sandbox/delete/route.ts') const hostCommandsPath = path.join(root, 'app/lib/hostCommands.ts') const telemetryRoutePath = path.join(root, 'app/api/telemetry/real/route.ts') const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts') +const inferenceEndpointUrlPath = path.join(root, 'app/lib/inferenceEndpointUrl.ts') const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts') const sandboxInferenceRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/route.ts') const sandboxInferenceApplyRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/apply/route.ts') @@ -28,6 +29,7 @@ const deleteRouteSource = await readFile(deleteRoutePath, 'utf8') const hostCommandsSource = await readFile(hostCommandsPath, 'utf8') const telemetryRouteSource = await readFile(telemetryRoutePath, 'utf8') const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8') +const inferenceEndpointUrlSource = await readFile(inferenceEndpointUrlPath, 'utf8') const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8') const sandboxInferenceRouteSource = await readFile(sandboxInferenceRoutePath, 'utf8') const sandboxInferenceApplyRouteSource = await readFile(sandboxInferenceApplyRoutePath, 'utf8') @@ -91,7 +93,11 @@ assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route') assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM-compatible endpoints') assert.match(inferenceRouteSource, /normalizeProviderType\(requestedType, name, baseUrl\)/, 'inference route must force vLLM endpoints through the OpenAI-compatible provider type') -assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config') +assert.match(inferenceRouteSource, /normalizeInferenceBaseUrlForGateway\(\s*baseUrl,/, 'inference route must normalize endpoint URLs for the OpenShell gateway pod') +assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{normalizedBaseUrl\}/, 'inference route must pass the normalized endpoint URL as provider config') +assert.match(createRouteSource, /normalizeInferenceBaseUrlForGateway\(endpointUrl, "openai"\)/, 'sandbox create must normalize env-derived OpenAI-compatible endpoints before NemoClaw onboarding') +assert.match(inferenceEndpointUrlSource, /host\.docker\.internal/, 'endpoint normalization must rewrite host-local URLs to a gateway-reachable host alias') +assert.match(inferenceEndpointUrlSource, /\/chat\/completions/, 'endpoint normalization must strip concrete OpenAI endpoint suffixes to the base URL') assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration') assert.doesNotMatch(inferencePanelSource, /NVIDIA Preset|useNvidiaInferencePreset|aws\/anthropic\/claude-opus-4-5/, 'settings UI must not expose NVIDIA-corp-specific inference presets') assert.match(inferencePanelSource, /useVllmPreset/, 'settings UI must expose a vLLM endpoint preset') @@ -119,6 +125,7 @@ assert.match(sandboxInferencePanelSource, /Add Route/, 'selected sandbox UI must assert.match(sandboxInferenceApplyLibSource, /\/sandbox\/\.openclaw\/openclaw\.json/, 'live apply must patch OpenClaw config inside the running sandbox') assert.match(sandboxInferenceApplyLibSource, /sha256sum \/sandbox\/\.openclaw\/openclaw\.json > \/sandbox\/\.openclaw\/\.config-hash/, 'live apply must refresh the immutable OpenClaw config hash') assert.match(sandboxInferenceApplyLibSource, /\["inference", "set", "--no-verify", "--provider", primary\.provider, "--model", primary\.model\]/, 'live apply must point OpenShell inference at the primary route') +assert.match(sandboxInferenceApplyLibSource, /name: route\.model/, 'live apply must keep model display names raw while primary refs remain provider-qualified') assert.doesNotMatch(sandboxRestartRouteSource, /delete", "pod"|kubectl delete/, 'restart route must not delete the backing sandbox pod') assert.match(sandboxRestartRouteSource, /restartMode: "openclaw-runtime"/, 'restart route must restart only the in-sandbox OpenClaw runtime') assert.match(sandboxRestartRouteSource, /waitForSandboxReady\(sandboxName, 15000, 1000\)/, 'restart route must verify OpenShell reports the sandbox ready before runtime restart')