Add chat sidecar and verified inference routes

This commit is contained in:
mmckeen-nv
2026-05-13 11:02:26 -07:00
parent 2f51629324
commit b3c4df18ea
13 changed files with 1766 additions and 137 deletions
+67 -6
View File
@@ -3,6 +3,14 @@ import { execFile } from "node:child_process"
import { promisify } from "node:util"
import { normalizeInferenceBaseUrlForGateway } from "@/app/lib/inferenceEndpointUrl"
import { OPENSHELL_BIN, hostCommandEnv } from "@/app/lib/hostCommands"
import {
listSavedVerifiedInferenceRoutes,
mergeVerifiedInferenceRoutes,
rememberVerifiedInferenceRoute,
removeVerifiedInferenceRoutesForProvider,
type VerifiedInferenceRoute,
} from "@/app/lib/inferenceRouteStore"
import { listSandboxInferenceRoutes } from "@/app/lib/sandboxInferenceStore"
const execFileAsync = promisify(execFile)
@@ -16,6 +24,10 @@ type InferenceRoute = {
timeout: string | null
}
type ProviderSummary = {
name: string | null
}
function stripAnsi(value: string) {
return value.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "")
}
@@ -53,6 +65,39 @@ function parseRoute(output: string, label: "Gateway inference" | "System inferen
}
}
function verifiedRouteFromStatus(route: InferenceRoute, label: string, source: "gateway" | "system"): VerifiedInferenceRoute | null {
if (!route.configured || !route.provider || !route.model) return null
return {
id: `${route.provider}::${route.model}`,
provider: route.provider,
model: route.model,
label,
source,
lastVerifiedAt: null,
}
}
async function buildVerifiedRoutes(gateway: InferenceRoute, system: InferenceRoute, providers: ProviderSummary[] = []) {
const [savedRoutes, sandboxRoutes] = await Promise.all([
listSavedVerifiedInferenceRoutes(),
listSandboxInferenceRoutes(),
])
const providerNames = new Set(providers.map((provider) => provider.name).filter(Boolean))
const providerIsConfigured = (provider: string) => providerNames.size === 0 || providerNames.has(provider)
return mergeVerifiedInferenceRoutes([
...sandboxRoutes.filter((route) => providerIsConfigured(route.provider)).map((route) => ({
provider: route.provider,
model: route.model,
label: route.label || "Saved sandbox route",
source: "sandbox",
lastVerifiedAt: null,
})),
...savedRoutes.filter((route) => providerIsConfigured(route.provider)),
verifiedRouteFromStatus(system, "Active system route", "system"),
verifiedRouteFromStatus(gateway, "Active sandbox route", "gateway"),
])
}
function parseProvider(output: string) {
return {
id: readField(output, "Id"),
@@ -128,10 +173,13 @@ export async function GET() {
runOpenShell(["inference", "get"]),
readProviders(),
])
const gateway = parseRoute(inference.stdout, "Gateway inference")
const system = parseRoute(inference.stdout, "System inference")
return NextResponse.json({
ok: true,
gateway: parseRoute(inference.stdout, "Gateway inference"),
system: parseRoute(inference.stdout, "System inference"),
gateway,
system,
verifiedRoutes: await buildVerifiedRoutes(gateway, system, providers),
providers,
raw: inference.stdout,
})
@@ -182,15 +230,24 @@ export async function POST(request: Request) {
if (noVerify) routeArgs.push("--no-verify")
if (Number.isFinite(timeout) && timeout >= 0) routeArgs.push("--timeout", String(timeout))
inferenceResult = await runOpenShell(routeArgs)
await rememberVerifiedInferenceRoute({
provider: name,
model,
label: system ? "System inference" : "Sandbox inference",
source: system ? "system" : "gateway",
})
}
const refreshed = await runOpenShell(["inference", "get"])
const gateway = parseRoute(refreshed.stdout, "Gateway inference")
const systemRoute = parseRoute(refreshed.stdout, "System inference")
return NextResponse.json({
ok: true,
provider: name,
active: makeActive,
gateway: parseRoute(refreshed.stdout, "Gateway inference"),
system: parseRoute(refreshed.stdout, "System inference"),
gateway,
system: systemRoute,
verifiedRoutes: await buildVerifiedRoutes(gateway, systemRoute),
stdout: redactSecretOutput([providerResult.stdout, inferenceResult?.stdout].filter(Boolean).join("\n\n"), [apiKey]),
stderr: redactSecretOutput([providerResult.stderr, inferenceResult?.stderr].filter(Boolean).join("\n\n"), [apiKey]),
})
@@ -206,16 +263,20 @@ export async function DELETE(request: Request) {
const url = new URL(request.url)
const name = validateName(body?.name ?? url.searchParams.get("name"))
const result = await runOpenShell(["provider", "delete", name])
await removeVerifiedInferenceRoutesForProvider(name)
const [inference, providers] = await Promise.all([
runOpenShell(["inference", "get"]),
readProviders(),
])
const gateway = parseRoute(inference.stdout, "Gateway inference")
const system = parseRoute(inference.stdout, "System inference")
return NextResponse.json({
ok: true,
provider: name,
gateway: parseRoute(inference.stdout, "Gateway inference"),
system: parseRoute(inference.stdout, "System inference"),
gateway,
system,
verifiedRoutes: await buildVerifiedRoutes(gateway, system, providers),
providers,
stdout: result.stdout,
stderr: result.stderr,
+87 -33
View File
@@ -1,6 +1,6 @@
"use client"
import { useCallback, useEffect, useMemo, useState } from "react"
import { useCallback, useEffect, useState } from "react"
import type { SandboxInventoryItem } from "../hooks/useSandboxInventory"
type ProviderSummary = {
@@ -32,6 +32,21 @@ type SandboxInferenceRoute = {
label: string
}
type VerifiedInferenceRoute = {
id?: string
provider: string | null
model: string | null
label?: string | null
source?: string | null
lastVerifiedAt?: string | null
}
type VerifiedSandboxInferenceRoute = SandboxInferenceRoute & {
scope: string
source: string
lastVerifiedAt: string | null
}
type SandboxInferenceConfig = {
sandboxId: string
provider: string
@@ -65,6 +80,49 @@ function dedupeRoutes(routes: SandboxInferenceRoute[]) {
})).values())
}
function sourceScope(source: string | null | undefined) {
switch (source) {
case "gateway":
return "Sandbox"
case "system":
return "System"
case "sandbox":
return "Sandbox Config"
default:
return "Verified"
}
}
function verifiedRouteFromStatus(route: InferenceRouteStatus | undefined, label: string, source: "gateway" | "system") {
return route?.configured && route.provider && route.model
? {
...makeRoute(route.provider, route.model, label),
scope: sourceScope(source),
source,
lastVerifiedAt: null,
}
: null
}
function normalizeVerifiedRoute(route: VerifiedInferenceRoute): VerifiedSandboxInferenceRoute | null {
const provider = typeof route.provider === "string" ? route.provider.trim() : ""
const model = typeof route.model === "string" ? route.model.trim() : ""
if (!provider || !model) return null
const source = typeof route.source === "string" && route.source.trim() ? route.source.trim() : "saved"
return {
...makeRoute(provider, model, typeof route.label === "string" ? route.label.trim() : ""),
scope: sourceScope(source),
source,
lastVerifiedAt: typeof route.lastVerifiedAt === "string" && route.lastVerifiedAt.trim() ? route.lastVerifiedAt : null,
}
}
function dedupeVerifiedRoutes(routes: Array<VerifiedSandboxInferenceRoute | null>) {
return Array.from(new Map(routes
.filter((route): route is VerifiedSandboxInferenceRoute => Boolean(route))
.map((route) => [route.id, route])).values())
}
export default function SandboxInferencePanel({
sandbox,
embedded = false,
@@ -79,7 +137,7 @@ export default function SandboxInferencePanel({
const [applying, setApplying] = useState(false)
const [message, setMessage] = useState("")
const [providers, setProviders] = useState<ProviderSummary[]>([])
const [verifiedRoutes, setVerifiedRoutes] = useState<Array<SandboxInferenceRoute & { scope: "Sandbox" | "System" }>>([])
const [verifiedRoutes, setVerifiedRoutes] = useState<VerifiedSandboxInferenceRoute[]>([])
const [routes, setRoutes] = useState<SandboxInferenceRoute[]>([])
const [primaryRouteId, setPrimaryRouteId] = useState("")
const [draftProvider, setDraftProvider] = useState("")
@@ -92,10 +150,6 @@ export default function SandboxInferencePanel({
const draftProviderIsOllama = draftProvider.toLowerCase().includes("ollama")
const anyOllamaRoute = routes.some((route) => route.provider.toLowerCase().includes("ollama"))
const shouldPollOllama = draftProviderIsOllama || anyOllamaRoute
const primaryRoute = useMemo(
() => routes.find((route) => route.id === primaryRouteId) || routes[0] || null,
[routes, primaryRouteId]
)
const loadOllamaModels = useCallback(async () => {
try {
@@ -125,16 +179,21 @@ export default function SandboxInferencePanel({
const config = configData.config as SandboxInferenceConfig
const gatewayRoute = inferenceData.gateway as InferenceRouteStatus | undefined
const systemRoute = inferenceData.system as InferenceRouteStatus | undefined
const nextVerifiedRoutes = [
gatewayRoute?.configured && gatewayRoute.provider && gatewayRoute.model
? { ...makeRoute(gatewayRoute.provider, gatewayRoute.model, "Active sandbox route"), scope: "Sandbox" as const }
: null,
systemRoute?.configured && systemRoute.provider && systemRoute.model
? { ...makeRoute(systemRoute.provider, systemRoute.model, "Active system route"), scope: "System" as const }
: null,
].filter((route): route is SandboxInferenceRoute & { scope: "Sandbox" | "System" } => Boolean(route))
const fallbackProvider = gatewayRoute?.provider || nextProviders[0]?.name || ""
const fallbackModel = gatewayRoute?.model || ""
const explicitVerifiedRoutes = Array.isArray(inferenceData.verifiedRoutes)
? inferenceData.verifiedRoutes.map((route: VerifiedInferenceRoute) => normalizeVerifiedRoute(route))
: []
const hasExplicitVerifiedRoutes = explicitVerifiedRoutes.some(Boolean)
const nextVerifiedRoutes = dedupeVerifiedRoutes(hasExplicitVerifiedRoutes
? explicitVerifiedRoutes
: [
verifiedRouteFromStatus(gatewayRoute, "Active sandbox route", "gateway"),
verifiedRouteFromStatus(systemRoute, "Active system route", "system"),
])
const fallbackRoute = gatewayRoute?.configured && gatewayRoute.provider && gatewayRoute.model
? makeRoute(gatewayRoute.provider, gatewayRoute.model, "Gateway default")
: nextVerifiedRoutes[0]
const fallbackProvider = fallbackRoute?.provider || nextProviders[0]?.name || ""
const fallbackModel = fallbackRoute?.model || ""
const nextRoutes = dedupeRoutes(Array.isArray(config?.routes) && config.routes.length > 0
? config.routes
: fallbackProvider && fallbackModel
@@ -298,24 +357,18 @@ export default function SandboxInferencePanel({
</button>
</div>
)}
{primaryRoute && (
<div className="metric p-4">
<p className="text-[10px] uppercase tracking-wider text-[var(--foreground-dim)]">Primary Route</p>
<p className="mt-2 text-sm font-mono text-[var(--foreground)]">{primaryRoute.provider}</p>
<p className="mt-1 text-xs font-mono text-[var(--foreground-dim)]">{primaryRoute.model}</p>
<div className="rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] p-4">
<div className="flex items-center justify-between gap-4">
<h5 className="text-xs uppercase tracking-wider text-[var(--foreground)]">Verified Working Routes</h5>
<span className="text-[10px] uppercase tracking-wider text-[var(--nvidia-green)]">{verifiedRoutes.length} Available</span>
</div>
)}
{verifiedRoutes.length > 0 && (
<div className="rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] p-4">
<div className="flex items-center justify-between gap-4">
<h5 className="text-xs uppercase tracking-wider text-[var(--foreground)]">Verified Working Routes</h5>
<span className="text-[10px] uppercase tracking-wider text-[var(--nvidia-green)]">OpenShell active</span>
</div>
{verifiedRoutes.length === 0 ? (
<p className="mt-3 text-xs text-[var(--foreground-dim)]">No verified routes reported by the main inference config.</p>
) : (
<div className="mt-3 grid grid-cols-1 gap-2 md:grid-cols-2">
{verifiedRoutes.map((route) => (
<button
key={`${route.scope}-${route.id}`}
key={`${route.source}-${route.id}`}
type="button"
onClick={() => addRoute(route.provider, route.model, route.label)}
className="rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] p-3 text-left hover:border-[var(--nvidia-green)]"
@@ -326,11 +379,12 @@ export default function SandboxInferencePanel({
</div>
<div className="mt-2 text-xs font-mono text-[var(--foreground)]">{route.provider}</div>
<div className="mt-1 break-all text-[11px] font-mono text-[var(--foreground-dim)]">{route.model}</div>
{route.label && <div className="mt-2 text-[10px] uppercase tracking-wider text-[var(--foreground-dim)]">{route.label}</div>}
</button>
))}
</div>
</div>
)}
)}
</div>
<div className="grid grid-cols-1 lg:grid-cols-[minmax(180px,240px)_minmax(0,1fr)_minmax(160px,220px)_auto] gap-3 items-end">
<div className="space-y-2">
@@ -388,7 +442,7 @@ export default function SandboxInferencePanel({
<div key={route.id} className="grid grid-cols-1 lg:grid-cols-[auto_minmax(160px,220px)_minmax(0,1fr)_minmax(140px,200px)_auto] gap-3 rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] p-3 items-center">
<label className="flex items-center gap-2 text-xs text-[var(--foreground-dim)]">
<input type="radio" checked={primaryRouteId === route.id} onChange={() => setPrimaryRouteId(route.id)} />
Primary
Default
</label>
<span className="text-xs font-mono text-[var(--foreground)]">{route.provider}</span>
<input value={route.model} onChange={(event) => updateRoute(route.id, { model: event.target.value })} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background)] px-3 py-2 text-xs font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" />
+123
View File
@@ -0,0 +1,123 @@
import { mkdir, readFile, writeFile } from "node:fs/promises"
import path from "node:path"
export type VerifiedInferenceRoute = {
id: string
provider: string
model: string
label: string
source: string
lastVerifiedAt: string | null
}
type StoreShape = {
routes: Record<string, VerifiedInferenceRoute>
}
const HOME = process.env.HOME || "/tmp"
const STORE_DIR = process.env.NEMOCLAW_DASHBOARD_STATE_DIR || path.join(HOME, ".nemoclaw-dashboard")
const STORE_PATH = path.join(STORE_DIR, "inference-routes.json")
function cleanString(value: unknown) {
return typeof value === "string" ? value.trim() : ""
}
function emptyStore(): StoreShape {
return { routes: {} }
}
async function readStore(): Promise<StoreShape> {
try {
const raw = await readFile(STORE_PATH, "utf8")
const parsed = JSON.parse(raw)
return parsed && typeof parsed === "object" && parsed.routes && typeof parsed.routes === "object"
? parsed
: emptyStore()
} catch {
return emptyStore()
}
}
async function writeStore(store: StoreShape) {
await mkdir(STORE_DIR, { recursive: true, mode: 0o700 })
await writeFile(STORE_PATH, JSON.stringify(store, null, 2), { mode: 0o600 })
}
export function inferenceRouteId(provider: string, model: string) {
return `${provider}::${model}`
}
export function normalizeVerifiedInferenceRoute(input: Partial<VerifiedInferenceRoute> | null | undefined): VerifiedInferenceRoute | null {
const provider = cleanString(input?.provider)
const model = cleanString(input?.model)
if (!provider || !model) return null
return {
id: inferenceRouteId(provider, model),
provider,
model,
label: cleanString(input?.label) || "Verified route",
source: cleanString(input?.source) || "saved",
lastVerifiedAt: cleanString(input?.lastVerifiedAt) || null,
}
}
export function mergeVerifiedInferenceRoutes(inputs: Array<Partial<VerifiedInferenceRoute> | null | undefined>) {
const sourceRank: Record<string, number> = {
sandbox: 1,
saved: 2,
system: 3,
gateway: 4,
}
const routes = new Map<string, VerifiedInferenceRoute>()
for (const input of inputs) {
const route = normalizeVerifiedInferenceRoute(input)
if (!route) continue
const existing = routes.get(route.id)
if (!existing) {
routes.set(route.id, route)
continue
}
const existingRank = sourceRank[existing.source] || 0
const routeRank = sourceRank[route.source] || 0
routes.set(route.id, routeRank >= existingRank ? { ...existing, ...route } : existing)
}
return Array.from(routes.values()).sort((first, second) => {
const rankDelta = (sourceRank[second.source] || 0) - (sourceRank[first.source] || 0)
if (rankDelta !== 0) return rankDelta
return `${first.provider}/${first.model}`.localeCompare(`${second.provider}/${second.model}`)
})
}
export async function listSavedVerifiedInferenceRoutes() {
const store = await readStore()
return mergeVerifiedInferenceRoutes(Object.values(store.routes))
}
export async function rememberVerifiedInferenceRoute(input: Partial<VerifiedInferenceRoute>) {
const route = normalizeVerifiedInferenceRoute({
...input,
lastVerifiedAt: input.lastVerifiedAt || new Date().toISOString(),
})
if (!route) return null
const store = await readStore()
store.routes[route.id] = route
await writeStore(store)
return route
}
export async function removeVerifiedInferenceRoutesForProvider(provider: string) {
const cleanProvider = provider.trim()
if (!cleanProvider) return
const store = await readStore()
for (const [id, route] of Object.entries(store.routes)) {
if (route.provider === cleanProvider) delete store.routes[id]
}
await writeStore(store)
}
+31 -2
View File
@@ -67,7 +67,7 @@ export const MCP_SERVER_CATALOG: McpCatalogEntry[] = [
{
id: "inter-sandbox-chat",
name: "Inter-Sandbox Chat",
summary: "Give allowed sandboxes a shared roomed message board so agents can post updates and read messages from each other through the MCP broker.",
summary: "Give allowed sandboxes a shared roomed message board. Agents acknowledge only operator-originated chats; sandbox-to-sandbox chatter stays receipt-free.",
transport: "stdio",
command: "node",
args: [path.join(process.cwd(), "scripts/inter-sandbox-chat-mcp.mjs")],
@@ -95,7 +95,7 @@ export const MCP_SERVER_CATALOG: McpCatalogEntry[] = [
tags: ["web", "utility"],
},
]
const BASELINE_MCP_SERVER_IDS = ["memory"]
const BASELINE_MCP_SERVER_IDS = ["memory", "inter-sandbox-chat"]
function emptyStore(): StoreShape {
return { servers: {} }
@@ -196,6 +196,34 @@ async function ensureBaselineMcpServers(store: StoreShape) {
if (changed) await writeStore(store)
}
async function refreshCatalogMcpServers(store: StoreShape) {
let changed = false
const catalogById = new Map(MCP_SERVER_CATALOG.map((entry) => [entry.id, entry]))
for (const [id, existing] of Object.entries(store.servers)) {
if (existing.source !== "catalog") continue
const catalog = catalogById.get(id)
if (!catalog) continue
const next = {
...existing,
name: catalog.name,
summary: catalog.summary,
websiteUrl: catalog.websiteUrl,
transport: catalog.transport,
command: catalog.command,
args: [...catalog.args],
env: { ...catalog.env },
tags: [...catalog.tags],
}
if (JSON.stringify(next) === JSON.stringify(existing)) continue
store.servers[id] = {
...next,
updatedAt: new Date().toISOString(),
}
changed = true
}
if (changed) await writeStore(store)
}
export function buildMcpClientConfig(servers: McpServerInstall[]) {
return {
mcpServers: Object.fromEntries(
@@ -228,6 +256,7 @@ export function sandboxCanAccessMcpServer(server: McpServerInstall, sandboxId: s
export async function listMcpServers() {
const store = await readStore()
await ensureBaselineMcpServers(store)
await refreshCatalogMcpServers(store)
const servers = Object.values(store.servers).sort((a, b) => a.name.localeCompare(b.name))
return {
catalog: MCP_SERVER_CATALOG,
+8
View File
@@ -93,6 +93,14 @@ export async function getSandboxInferenceConfig(sandboxId: string): Promise<Sand
return normalizeSandboxInferenceConfig(sandboxId, store.sandboxes[sandboxId])
}
export async function listSandboxInferenceRoutes(): Promise<SandboxInferenceRoute[]> {
const store = await readStore()
const routes = Object.entries(store.sandboxes)
.flatMap(([sandboxId, config]) => normalizeSandboxInferenceConfig(sandboxId, config).routes)
.filter((route) => route.enabled)
return Array.from(new Map(routes.map((route) => [route.id, route])).values())
}
export async function saveSandboxInferenceConfig(sandboxId: string, input: Partial<SandboxInferenceConfig>): Promise<SandboxInferenceConfig> {
const store = await readStore()
const next = normalizeSandboxInferenceConfig(sandboxId, {