Add vLLM advanced configuration
This commit is contained in:
@@ -8,6 +8,8 @@ const deleteRoutePath = path.join(root, 'app/api/sandbox/delete/route.ts')
|
||||
const hostCommandsPath = path.join(root, 'app/lib/hostCommands.ts')
|
||||
const telemetryRoutePath = path.join(root, 'app/api/telemetry/real/route.ts')
|
||||
const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts')
|
||||
const vllmApplyRoutePath = path.join(root, 'app/api/inference/vllm/apply/route.ts')
|
||||
const vllmRestartRoutePath = path.join(root, 'app/api/inference/vllm/restart/route.ts')
|
||||
const inferenceEndpointUrlPath = path.join(root, 'app/lib/inferenceEndpointUrl.ts')
|
||||
const ollamaModelsRoutePath = path.join(root, 'app/api/ollama/models/route.ts')
|
||||
const sandboxInferenceRoutePath = path.join(root, 'app/api/sandbox/[sandboxId]/inference/route.ts')
|
||||
@@ -33,6 +35,8 @@ const deleteRouteSource = await readFile(deleteRoutePath, 'utf8')
|
||||
const hostCommandsSource = await readFile(hostCommandsPath, 'utf8')
|
||||
const telemetryRouteSource = await readFile(telemetryRoutePath, 'utf8')
|
||||
const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8')
|
||||
const vllmApplyRouteSource = await readFile(vllmApplyRoutePath, 'utf8')
|
||||
const vllmRestartRouteSource = await readFile(vllmRestartRoutePath, 'utf8')
|
||||
const inferenceEndpointUrlSource = await readFile(inferenceEndpointUrlPath, 'utf8')
|
||||
const ollamaModelsRouteSource = await readFile(ollamaModelsRoutePath, 'utf8')
|
||||
const sandboxInferenceRouteSource = await readFile(sandboxInferenceRoutePath, 'utf8')
|
||||
@@ -176,11 +180,13 @@ assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--t
|
||||
assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference route must update existing OpenShell providers')
|
||||
assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference route must delete OpenShell providers')
|
||||
assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route')
|
||||
assert.match(inferenceRouteSource, /"vllm"/, 'inference route must accept the dashboard vLLM provider type')
|
||||
assert.match(inferenceRouteSource, /type === "vllm"[\s\S]*\? "openai" : type/, 'inference route must store vLLM selections as OpenAI-compatible providers')
|
||||
assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM-compatible endpoints')
|
||||
assert.match(inferenceRouteSource, /normalizeProviderType\(requestedType, name, baseUrl\)/, 'inference route must force vLLM endpoints through the OpenAI-compatible provider type')
|
||||
assert.match(inferenceRouteSource, /isOllamaEndpoint\(name, baseUrl\)/, 'inference route must detect local Ollama endpoints')
|
||||
assert.match(inferenceRouteSource, /isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must force local Ollama endpoints through the OpenAI-compatible provider type')
|
||||
assert.match(inferenceRouteSource, /body\?\.noVerify !== false \|\| isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must skip verification for local Ollama endpoints')
|
||||
assert.match(inferenceRouteSource, /body\?\.noVerify !== false \|\| requestedType === "vllm" \|\| isVllmEndpoint\(name, baseUrl\) \|\| isOllamaEndpoint\(name, baseUrl\)/, 'inference route must skip verification for vLLM selections and local Ollama endpoints')
|
||||
assert.match(inferenceRouteSource, /normalizeInferenceBaseUrlForGateway\(\s*baseUrl,/, 'inference route must normalize endpoint URLs for the OpenShell gateway pod')
|
||||
assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{normalizedBaseUrl\}/, 'inference route must pass the normalized endpoint URL as provider config')
|
||||
assert.match(inferenceEndpointUrlSource, /host\.docker\.internal/, 'endpoint normalization must rewrite host-local URLs to a gateway-reachable host alias')
|
||||
@@ -188,7 +194,35 @@ assert.match(inferenceEndpointUrlSource, /\/chat\/completions/, 'endpoint normal
|
||||
assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration')
|
||||
assert.doesNotMatch(inferencePanelSource, /NVIDIA Preset|useNvidiaInferencePreset|aws\/anthropic\/claude-opus-4-5/, 'settings UI must not expose NVIDIA-corp-specific inference presets')
|
||||
assert.match(inferencePanelSource, /useVllmPreset/, 'settings UI must expose a vLLM endpoint preset')
|
||||
assert.match(inferencePanelSource, /\{ value: "vllm", label: "vLLM" \}/, 'settings UI must expose vLLM as an explicit provider type')
|
||||
assert.match(inferencePanelSource, /setType\("vllm"\)/, 'vLLM preset must select the explicit vLLM provider type')
|
||||
assert.match(inferencePanelSource, /const isVllmEndpointSelected = type === "vllm"/, 'settings UI must only show advanced vLLM controls when the vLLM provider type is selected')
|
||||
assert.match(inferencePanelSource, /host\.docker\.internal:8000\/v1/, 'vLLM preset must point at the host vLLM OpenAI-compatible endpoint')
|
||||
assert.match(inferencePanelSource, /function isVllmEndpoint\(name: string, baseUrl: string\)/, 'settings UI must detect vLLM-compatible endpoint selections')
|
||||
assert.match(inferencePanelSource, /vLLM Advanced Configuration/, 'settings UI must expose advanced vLLM launch tuning when vLLM is selected')
|
||||
assert.match(inferencePanelSource, /\/api\/inference\/vllm\/apply/, 'vLLM advanced configuration must apply launch changes through the controller')
|
||||
assert.match(inferencePanelSource, /\/api\/inference\/vllm\/restart/, 'vLLM advanced configuration must expose a restart action')
|
||||
assert.match(inferencePanelSource, /--max-model-len/, 'vLLM advanced configuration must generate a max model length launch flag')
|
||||
assert.match(inferencePanelSource, /--gpu-memory-utilization/, 'vLLM advanced configuration must generate a GPU memory utilization launch flag')
|
||||
assert.match(inferencePanelSource, /--kv-cache-dtype/, 'vLLM advanced configuration must expose KV cache dtype tuning')
|
||||
assert.match(inferencePanelSource, /--hf-overrides/, 'vLLM advanced configuration must generate HF overrides for long-context model config')
|
||||
assert.match(inferencePanelSource, /rope_scaling/, 'vLLM advanced configuration must expose YaRN/RoPE scaling')
|
||||
assert.match(inferencePanelSource, /max_position_embeddings/, 'vLLM advanced configuration must override max_position_embeddings for long-context kernels')
|
||||
assert.match(inferencePanelSource, /YaRN\/RoPE scaling/, 'vLLM advanced configuration must let users toggle long-context scaling')
|
||||
assert.match(inferencePanelSource, /VLLM_ALLOW_LONG_MAX_MODEL_LEN=1/, 'vLLM advanced configuration must include the long max model length safety override for extended contexts')
|
||||
assert.match(inferencePanelSource, /--enable-auto-tool-choice/, 'vLLM advanced configuration must include the tool-choice launch flag')
|
||||
assert.match(inferencePanelSource, /--tool-call-parser/, 'vLLM advanced configuration must include the tool-call parser launch flag')
|
||||
assert.match(vllmApplyRouteSource, /\["rm", "-f", container\]/, 'vLLM apply route must remove the previous container before recreating it')
|
||||
assert.match(vllmApplyRouteSource, /"run", "-d"/, 'vLLM apply route must launch a replacement vLLM container')
|
||||
assert.match(vllmApplyRouteSource, /--hf-overrides/, 'vLLM apply route must pass long-context HF overrides to vLLM')
|
||||
assert.match(vllmApplyRouteSource, /rope_scaling/, 'vLLM apply route must include YaRN/RoPE scaling when enabled')
|
||||
assert.match(vllmApplyRouteSource, /max_position_embeddings/, 'vLLM apply route must expand max_position_embeddings with max model length')
|
||||
assert.match(vllmApplyRouteSource, /VLLM_ALLOW_LONG_MAX_MODEL_LEN=1/, 'vLLM apply route must allow vLLM to start above the derived 32K model length')
|
||||
assert.match(vllmApplyRouteSource, /execFileAsync\(DOCKER_BIN, runArgs/, 'vLLM apply route must execute Docker with argv instead of shell text')
|
||||
assert.match(vllmRestartRouteSource, /\["restart", container\]/, 'vLLM restart route must restart the selected vLLM container with docker restart')
|
||||
assert.match(vllmRestartRouteSource, /OPENSHELL_CONTROL_VLLM_CONTAINER/, 'vLLM restart route must support an explicit default container env var')
|
||||
assert.match(vllmRestartRouteSource, /OPENSHELL_CONTROL_VLLM_CONTAINERS|VLLM_CONTAINER_ALLOWLIST/, 'vLLM restart route must support an explicit container allowlist')
|
||||
assert.match(vllmRestartRouteSource, /\^vllm/, 'vLLM restart route must restrict generic restarts to vLLM-named containers')
|
||||
assert.doesNotMatch(inferencePanelSource, /nvidia\/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4/, 'settings UI must not default rebuilt inference routes to the NVIDIA Nemotron model')
|
||||
assert.match(inferencePanelSource, /method: "DELETE"/, 'settings UI must expose provider deletion')
|
||||
assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models')
|
||||
|
||||
Reference in New Issue
Block a user