Add chat sidecar and verified inference routes

This commit is contained in:
mmckeen-nv
2026-05-13 11:02:26 -07:00
parent 2f51629324
commit b3c4df18ea
13 changed files with 1766 additions and 137 deletions
+10
View File
@@ -11,6 +11,7 @@ const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts')
const vllmApplyRoutePath = path.join(root, 'app/api/inference/vllm/apply/route.ts')
const vllmRestartRoutePath = path.join(root, 'app/api/inference/vllm/restart/route.ts')
const inferenceEndpointUrlPath = path.join(root, 'app/lib/inferenceEndpointUrl.ts')
const inferenceRouteStorePath = path.join(root, 'app/lib/inferenceRouteStore.ts')
const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts')
const sandboxInferenceRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/route.ts')
const sandboxInferenceApplyRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/apply/route.ts')
@@ -38,6 +39,7 @@ const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8')
const vllmApplyRouteSource = await readFile(vllmApplyRoutePath, 'utf8')
const vllmRestartRouteSource = await readFile(vllmRestartRoutePath, 'utf8')
const inferenceEndpointUrlSource = await readFile(inferenceEndpointUrlPath, 'utf8')
const inferenceRouteStoreSource = await readFile(inferenceRouteStorePath, 'utf8')
const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8')
const sandboxInferenceRouteSource = await readFile(sandboxInferenceRoutePath, 'utf8')
const sandboxInferenceApplyRouteSource = await readFile(sandboxInferenceApplyRoutePath, 'utf8')
@@ -189,6 +191,11 @@ assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\) \|\| isOllam
assert.match(inferenceRouteSource, /body\?\.noVerify !== false \|\| requestedType === "vllm" \|\| isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM selections and local Ollama endpoints')
assert.match(inferenceRouteSource, /normalizeInferenceBaseUrlForGateway\(\s*baseUrl,/, 'inference route must normalize endpoint URLs for the OpenShell gateway pod')
assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{normalizedBaseUrl\}/, 'inference route must pass the normalized endpoint URL as provider config')
assert.match(inferenceRouteSource, /verifiedRoutes: await buildVerifiedRoutes\(gateway, system, providers\)/, 'inference route must expose a verified provider/model route catalog')
assert.match(inferenceRouteSource, /rememberVerifiedInferenceRoute/, 'inference route must remember successfully saved provider/model pairs')
assert.match(inferenceRouteSource, /listSandboxInferenceRoutes/, 'inference route must seed verified route choices from existing sandbox profiles')
assert.match(inferenceRouteStoreSource, /inference-routes\.json/, 'verified inference route store must persist dashboard-known route choices')
assert.match(inferenceRouteStoreSource, /mergeVerifiedInferenceRoutes/, 'verified inference route store must dedupe provider/model route choices')
assert.match(inferenceEndpointUrlSource, /host\.docker\.internal/, 'endpoint normalization must rewrite host-local URLs to a gateway-reachable host alias')
assert.match(inferenceEndpointUrlSource, /\/chat\/completions/, 'endpoint normalization must strip concrete OpenAI endpoint suffixes to the base URL')
assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration')
@@ -242,7 +249,10 @@ assert.match(sandboxInferenceStoreSource, /const id = `\$\{provider\}::\$\{model
assert.match(sandboxInferencePanelSource, /Save Sandbox Routes/, 'selected sandbox UI must expose multi-route inference saving')
assert.match(sandboxInferencePanelSource, /Apply to Running Container/, 'selected sandbox UI must expose live apply for running containers')
assert.match(sandboxInferencePanelSource, /Verified Working Routes/, 'selected sandbox UI must list verified active inference routes')
assert.match(sandboxInferencePanelSource, /inferenceData\.verifiedRoutes/, 'selected sandbox UI must read the shared verified route catalog')
assert.match(sandboxInferencePanelSource, /inferenceData\.system/, 'selected sandbox UI must include system route status when available')
assert.doesNotMatch(sandboxInferencePanelSource, /Primary Route/, 'selected sandbox UI must not render a separate primary route summary box')
assert.match(sandboxInferencePanelSource, /Default/, 'selected sandbox UI must keep route selection while avoiding a separate primary route card')
assert.match(sandboxInferencePanelSource, /\/api\/sandbox\/\$\{encodeURIComponent\(sandbox\.id\)\}\/inference/, 'selected sandbox UI must load and save sandbox-specific inference profiles')
assert.match(sandboxInferencePanelSource, /\/api\/sandbox\/\$\{encodeURIComponent\(sandbox\.id\)\}\/inference\/apply/, 'selected sandbox UI must call live apply endpoint')
assert.match(sandboxInferencePanelSource, /\/api\/ollama\/models/, 'selected sandbox UI must offer Ollama model selection')