Add default inference endpoint rollout

This commit is contained in:
Codex
2026-04-27 18:13:23 +00:00
parent 9603835db0
commit bd9f7a036a
3 changed files with 135 additions and 0 deletions
+67
View File
@@ -2,6 +2,8 @@ import { NextResponse } from "next/server"
import { execFile } from "node:child_process"
import { promisify } from "node:util"
import { HOST_PATH, OPENSHELL_BIN } from "@/app/lib/hostCommands"
import { applySandboxInferenceProfile } from "@/app/lib/sandboxInferenceApply"
import { saveSandboxInferenceConfig } from "@/app/lib/sandboxInferenceStore"
const execFileAsync = promisify(execFile)
@@ -72,6 +74,38 @@ function parseProvider(output: string) {
}
}
function parseSandboxNames(output: string) {
return output
.split(/\r?\n/)
.map((line) => line.trim())
.filter((line) => line && !/^name\s+/i.test(line) && !/^[-=]+$/.test(line) && !/^no sandboxes found\.?$/i.test(line))
.map((line) => line.split(/\s{2,}/)[0]?.trim())
.filter((name): name is string => Boolean(name))
}
async function readRunningSandboxes() {
const { stdout } = await runOpenShell(["sandbox", "list"])
const names = parseSandboxNames(stdout)
const sandboxes = []
for (const name of names) {
try {
const details = await runOpenShell(["sandbox", "get", name])
const rawPhase = readField(details.stdout, "Phase") || readField(details.stdout, "Status") || ""
if (!/^ready$/i.test(rawPhase)) continue
sandboxes.push({
name: readField(details.stdout, "Name") || name,
id: readField(details.stdout, "Id") || name,
phase: rawPhase,
})
} catch {
// Ignore sandboxes that disappear while the bulk route is being applied.
}
}
return sandboxes
}
async function readProviders() {
const { stdout } = await runOpenShell(["provider", "list", "--names"])
const names = stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean)
@@ -141,8 +175,10 @@ export async function POST(request: Request) {
const system = body?.route === "system" || body?.system === true
const noVerify = body?.noVerify !== false
const timeout = Number(body?.timeout ?? 0)
const applyToRunningSandboxes = body?.applyToRunningSandboxes === true
if (makeActive && !model) throw new Error("model is required when making the endpoint active")
if (applyToRunningSandboxes && system) throw new Error("default inference endpoint must target the sandbox route")
if (!/^[A-Z_][A-Z0-9_]*$/.test(credentialKey)) throw new Error("credential key must look like an environment variable name")
if (baseUrl) new URL(baseUrl)
@@ -165,6 +201,36 @@ export async function POST(request: Request) {
inferenceResult = await runOpenShell(routeArgs)
}
const sandboxApplyResults: any[] = []
if (applyToRunningSandboxes) {
const runningSandboxes = await readRunningSandboxes()
for (const sandbox of runningSandboxes) {
try {
await saveSandboxInferenceConfig(sandbox.id, {
provider: name,
primaryModel: model,
models: [model],
routes: [{
id: `${name}::${model}`,
provider: name,
model,
enabled: true,
label: "Default inference endpoint",
}],
primaryRouteId: `${name}::${model}`,
})
const applied = await applySandboxInferenceProfile(sandbox.id, sandbox.name)
sandboxApplyResults.push({ ok: true, sandbox, applied })
} catch (error) {
sandboxApplyResults.push({
ok: false,
sandbox,
error: error instanceof Error ? error.message : "Failed to apply default inference endpoint",
})
}
}
}
const refreshed = await runOpenShell(["inference", "get"])
return NextResponse.json({
ok: true,
@@ -172,6 +238,7 @@ export async function POST(request: Request) {
active: makeActive,
gateway: parseRoute(refreshed.stdout, "Gateway inference"),
system: parseRoute(refreshed.stdout, "System inference"),
sandboxApplyResults,
stdout: [providerResult.stdout, inferenceResult?.stdout].filter(Boolean).join("\n\n"),
stderr: [providerResult.stderr, inferenceResult?.stderr].filter(Boolean).join("\n\n"),
})
+62
View File
@@ -69,6 +69,8 @@ export default function InferenceEndpointPanel() {
const [ollamaLoading, setOllamaLoading] = useState(false)
const [ollamaMessage, setOllamaMessage] = useState("")
const [deletingProvider, setDeletingProvider] = useState<string | null>(null)
const [applyingDefault, setApplyingDefault] = useState(false)
const [confirmDefaultOpen, setConfirmDefaultOpen] = useState(false)
const activeProvider = useMemo(
() => providers.find((provider) => provider.name === gateway.provider) ?? null,
@@ -186,6 +188,45 @@ export default function InferenceEndpointPanel() {
}
}
async function setDefaultForAllRunningSandboxes() {
try {
setConfirmDefaultOpen(false)
setApplyingDefault(true)
setMessage("")
const response = await fetch("/api/inference", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
name,
type,
model,
baseUrl,
credentialKey,
apiKey,
route: "gateway",
timeout: Number(timeout || "0"),
noVerify,
makeActive: true,
applyToRunningSandboxes: true,
}),
})
const data = await response.json()
if (!response.ok) throw new Error(data.error || "Failed to set default inference endpoint")
const results = Array.isArray(data.sandboxApplyResults) ? data.sandboxApplyResults : []
const applied = results.filter((item: any) => item?.ok).length
const failed = results.length - applied
setGateway(data.gateway ?? gateway)
setSystem(data.system ?? system)
setApiKey("")
setMessage(`Default inference endpoint set to '${name}'. Applied to ${applied} running sandbox${applied === 1 ? "" : "es"}${failed ? `; ${failed} failed.` : "."}`)
await load()
} catch (error) {
setMessage(error instanceof Error ? error.message : "Failed to set default inference endpoint")
} finally {
setApplyingDefault(false)
}
}
async function deleteProvider(providerName: string | null) {
if (!providerName) return
if (!window.confirm(`Delete inference provider '${providerName}'?`)) return
@@ -333,6 +374,9 @@ export default function InferenceEndpointPanel() {
<button onClick={saveEndpoint} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-white text-xs font-mono uppercase tracking-wider disabled:opacity-50">
{saving ? "Saving..." : "Save Endpoint"}
</button>
<button type="button" onClick={() => setConfirmDefaultOpen(true)} disabled={saving || applyingDefault || !name || !model} className="px-4 py-2 rounded-sm border border-[var(--nvidia-green)] text-[var(--nvidia-green)] text-xs font-mono uppercase tracking-wider hover:bg-[rgba(118,185,0,0.08)] disabled:opacity-50">
{applyingDefault ? "Applying..." : "Set as Default Inference Endpoint"}
</button>
<button type="button" onClick={useNvidiaInferencePreset} disabled={saving} className="px-4 py-2 rounded-sm bg-[var(--background-tertiary)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:bg-[var(--background-secondary)] disabled:opacity-50">
NVIDIA Preset
</button>
@@ -366,6 +410,24 @@ export default function InferenceEndpointPanel() {
</div>
</div>
)}
{confirmDefaultOpen && (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 p-4">
<div className="w-full max-w-md rounded-sm border border-[var(--border-subtle)] bg-[var(--background-panel)] p-6 shadow-[var(--shadow-soft)]">
<h3 className="text-sm font-semibold uppercase tracking-wider text-[var(--foreground)]">Set Default Inference Endpoint</h3>
<p className="mt-3 text-sm text-[var(--foreground-dim)]">
This will change the default inference endpoint to ALL running sandboxes and newly created sandboxes.
</p>
<div className="mt-6 flex justify-end gap-3">
<button type="button" onClick={() => setConfirmDefaultOpen(false)} className="px-4 py-2 rounded-sm bg-[var(--background-tertiary)] text-[var(--foreground)] text-xs font-mono uppercase tracking-wider hover:bg-[var(--background-secondary)]">
No
</button>
<button type="button" onClick={setDefaultForAllRunningSandboxes} className="px-4 py-2 rounded-sm bg-[var(--nvidia-green)] text-black text-xs font-mono uppercase tracking-wider">
Yes
</button>
</div>
</div>
</div>
)}
</div>
)
}
+6
View File
@@ -74,10 +74,16 @@ assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference
assert.match(inferenceRouteSource, /\["provider", "delete", name\]/, 'inference route must delete OpenShell providers')
assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route')
assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config')
assert.match(inferenceRouteSource, /applyToRunningSandboxes/, 'inference route must support applying a default endpoint to running sandboxes')
assert.match(inferenceRouteSource, /readRunningSandboxes/, 'inference route must discover running sandboxes for default endpoint rollout')
assert.match(inferenceRouteSource, /applySandboxInferenceProfile/, 'inference route must live-apply the default endpoint to running sandboxes')
assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration')
assert.match(inferencePanelSource, /https:\/\/inference-api\.nvidia\.com\/v1/, 'settings UI must provide the NVIDIA inference API preset')
assert.match(inferencePanelSource, /aws\/anthropic\/claude-opus-4-5/, 'settings UI must include the requested NVIDIA inference model preset')
assert.match(inferencePanelSource, /method: "DELETE"/, 'settings UI must expose provider deletion')
assert.match(inferencePanelSource, /Set as Default Inference Endpoint/, 'settings UI must expose a default endpoint rollout action')
assert.match(inferencePanelSource, /This will change the default inference endpoint to ALL running sandboxes and newly created sandboxes\./, 'default endpoint action must warn about all running and new sandboxes')
assert.match(inferencePanelSource, /applyToRunningSandboxes: true/, 'settings UI must request default endpoint rollout after confirmation')
assert.match(inferencePanelSource, /\/api\/ollama\/models/, 'settings UI must poll local Ollama models')
assert.match(inferencePanelSource, /window\.setInterval\(\(\) => loadOllamaModels\(\{ silent: true \}\), 10000\)/, 'settings UI must autopoll Ollama while selected')
assert.match(ollamaModelsRouteSource, /\/api\/tags/, 'Ollama models route must query the Ollama tags API')