diff --git a/app/api/inference/route.ts b/app/api/inference/route.ts new file mode 100644 index 0000000..3b78154 --- /dev/null +++ b/app/api/inference/route.ts @@ -0,0 +1,201 @@ +import { NextResponse } from "next/server" +import { execFile } from "node:child_process" +import { existsSync } from "node:fs" +import { promisify } from "node:util" + +const execFileAsync = promisify(execFile) +const HOME = process.env.HOME || "" +const OPENSHELL_BIN_CANDIDATES = [ + process.env.OPENSHELL_BIN, + HOME ? `${HOME}/.local/bin/openshell` : undefined, + "/usr/local/bin/openshell", + "/opt/homebrew/bin/openshell", +].filter((value): value is string => Boolean(value)) +const OPENSHELL_BIN = + OPENSHELL_BIN_CANDIDATES.find((candidate) => existsSync(candidate)) ?? OPENSHELL_BIN_CANDIDATES[0] +const HOST_PATH = [ + `${HOME}/.local/bin`, + `${HOME}/.nvm/versions/node/v22.22.2/bin`, + `${HOME}/.nvm/versions/node/v22.22.1/bin`, + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + process.env.PATH || "", +].filter(Boolean).join(":") + +const PROVIDER_TYPES = new Set(["openai", "anthropic", "nvidia", "generic", "claude", "opencode", "codex", "copilot", "gitlab", "github", "outlook"]) + +type InferenceRoute = { + configured: boolean + provider: string | null + model: string | null + version: string | null + timeout: string | null +} + +function stripAnsi(value: string) { + return value.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "") +} + +async function runOpenShell(args: string[]) { + const { stdout, stderr } = await execFileAsync(OPENSHELL_BIN, args, { + env: { + ...process.env, + PATH: HOST_PATH, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY || "nemoclaw", + NO_COLOR: "1", + CLICOLOR: "0", + CLICOLOR_FORCE: "0", + }, + timeout: 60000, + maxBuffer: 20 * 1024 * 1024, + }) + return { stdout: stripAnsi(String(stdout).trim()), stderr: stripAnsi(String(stderr).trim()) } +} + +function readField(output: string, label: string) { + const match = output.match(new RegExp(`^\\s*${label}:\\s*(.+?)\\s*$`, "im")) + return match ? match[1].trim() : null +} + +function parseRoute(output: string, label: "Gateway inference" | "System inference"): InferenceRoute { + const otherLabel = label === "Gateway inference" ? "System inference" : "Gateway inference" + const pattern = new RegExp(`${label}:\\s*([\\s\\S]*?)(?:\\n\\s*${otherLabel}:|$)`, "i") + const section = output.match(pattern)?.[1] ?? "" + if (!section || /Not configured/i.test(section)) { + return { configured: false, provider: null, model: null, version: null, timeout: null } + } + return { + configured: true, + provider: readField(section, "Provider"), + model: readField(section, "Model"), + version: readField(section, "Version"), + timeout: readField(section, "Timeout"), + } +} + +function parseProvider(output: string) { + return { + id: readField(output, "Id"), + name: readField(output, "Name"), + type: readField(output, "Type"), + credentialKeys: (readField(output, "Credential keys") || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + configKeys: (readField(output, "Config keys") || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + } +} + +async function readProviders() { + const { stdout } = await runOpenShell(["provider", "list", "--names"]) + const names = stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean) + const providers = await Promise.all(names.map(async (name) => { + try { + const details = await runOpenShell(["provider", "get", name]) + return parseProvider(details.stdout) + } catch { + return { id: null, name, type: null, credentialKeys: [], configKeys: [] } + } + })) + return providers +} + +function validateName(name: unknown) { + if (typeof name !== "string" || !name.trim()) throw new Error("provider name is required") + const value = name.trim() + if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$/.test(value)) { + throw new Error("provider name must start with a letter or number and contain only letters, numbers, dots, underscores, or hyphens") + } + return value +} + +function validateProviderType(type: unknown) { + const value = typeof type === "string" && type.trim() ? type.trim() : "openai" + if (!PROVIDER_TYPES.has(value)) throw new Error(`unsupported provider type: ${value}`) + return value +} + +function optionalString(value: unknown) { + return typeof value === "string" ? value.trim() : "" +} + +function providerCredentialArg(credentialKey: string, apiKey: string) { + return apiKey ? `${credentialKey}=${apiKey}` : credentialKey +} + +export async function GET() { + try { + const [inference, providers] = await Promise.all([ + runOpenShell(["inference", "get"]), + readProviders(), + ]) + return NextResponse.json({ + ok: true, + gateway: parseRoute(inference.stdout, "Gateway inference"), + system: parseRoute(inference.stdout, "System inference"), + providers, + raw: inference.stdout, + }) + } catch (error) { + const message = error instanceof Error ? error.message : "Failed to read inference configuration" + return NextResponse.json({ ok: false, error: message }, { status: 500 }) + } +} + +export async function POST(request: Request) { + try { + const body = await request.json() + const name = validateName(body?.name) + const type = validateProviderType(body?.type) + const model = optionalString(body?.model) + const baseUrl = optionalString(body?.baseUrl) + const apiKey = optionalString(body?.apiKey) + const credentialKey = optionalString(body?.credentialKey) || "OPENAI_API_KEY" + const makeActive = body?.makeActive !== false + const system = body?.route === "system" || body?.system === true + const noVerify = body?.noVerify !== false + const timeout = Number(body?.timeout ?? 0) + + if (makeActive && !model) throw new Error("model is required when making the endpoint active") + if (!/^[A-Z_][A-Z0-9_]*$/.test(credentialKey)) throw new Error("credential key must look like an environment variable name") + if (baseUrl) new URL(baseUrl) + + const existingNames = (await readProviders()).map((provider) => provider.name).filter(Boolean) + const providerArgs = existingNames.includes(name) + ? ["provider", "update", name] + : ["provider", "create", "--name", name, "--type", type] + + if (!existingNames.includes(name) && !apiKey) providerArgs.push("--from-existing") + if (apiKey) providerArgs.push("--credential", providerCredentialArg(credentialKey, apiKey)) + if (baseUrl) providerArgs.push("--config", `OPENAI_BASE_URL=${baseUrl}`) + + const providerResult = await runOpenShell(providerArgs) + let inferenceResult = null + if (makeActive) { + const routeArgs = ["inference", "set", "--provider", name, "--model", model] + if (system) routeArgs.push("--system") + if (noVerify) routeArgs.push("--no-verify") + if (Number.isFinite(timeout) && timeout >= 0) routeArgs.push("--timeout", String(timeout)) + inferenceResult = await runOpenShell(routeArgs) + } + + const refreshed = await runOpenShell(["inference", "get"]) + return NextResponse.json({ + ok: true, + provider: name, + active: makeActive, + gateway: parseRoute(refreshed.stdout, "Gateway inference"), + system: parseRoute(refreshed.stdout, "System inference"), + stdout: [providerResult.stdout, inferenceResult?.stdout].filter(Boolean).join("\n\n"), + stderr: [providerResult.stderr, inferenceResult?.stderr].filter(Boolean).join("\n\n"), + }) + } catch (error) { + const message = error instanceof Error ? error.message : "Failed to save inference endpoint" + return NextResponse.json({ ok: false, error: message }, { status: /required|unsupported|must|URL/.test(message) ? 400 : 500 }) + } +} diff --git a/app/components/InferenceEndpointPanel.tsx b/app/components/InferenceEndpointPanel.tsx new file mode 100644 index 0000000..649350d --- /dev/null +++ b/app/components/InferenceEndpointPanel.tsx @@ -0,0 +1,247 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" + +type InferenceRoute = { + configured: boolean + provider: string | null + model: string | null + version: string | null + timeout: string | null +} + +type ProviderSummary = { + id: string | null + name: string | null + type: string | null + credentialKeys: string[] + configKeys: string[] +} + +type InferenceResponse = { + gateway: InferenceRoute + system: InferenceRoute + providers: ProviderSummary[] +} + +const providerTypeOptions = [ + { value: "openai", label: "OpenAI compatible" }, + { value: "nvidia", label: "NVIDIA" }, + { value: "anthropic", label: "Anthropic" }, + { value: "generic", label: "Generic" }, +] + +function emptyRoute(): InferenceRoute { + return { configured: false, provider: null, model: null, version: null, timeout: null } +} + +function FieldLabel({ children }: { children: React.ReactNode }) { + return +} + +export default function InferenceEndpointPanel() { + const [loading, setLoading] = useState(true) + const [saving, setSaving] = useState(false) + const [message, setMessage] = useState("") + const [providers, setProviders] = useState([]) + const [gateway, setGateway] = useState(emptyRoute) + const [system, setSystem] = useState(emptyRoute) + const [name, setName] = useState("nvidia-nim") + const [type, setType] = useState("openai") + const [model, setModel] = useState("nvidia/nemotron-3-super-120b-a12b") + const [baseUrl, setBaseUrl] = useState("https://integrate.api.nvidia.com/v1") + const [credentialKey, setCredentialKey] = useState("NVIDIA_API_KEY") + const [apiKey, setApiKey] = useState("") + const [route, setRoute] = useState<"gateway" | "system">("gateway") + const [timeout, setTimeoutValue] = useState("0") + const [noVerify, setNoVerify] = useState(true) + + const activeProvider = useMemo( + () => providers.find((provider) => provider.name === gateway.provider) ?? null, + [providers, gateway.provider] + ) + + async function load() { + try { + setLoading(true) + setMessage("") + const response = await fetch("/api/inference", { cache: "no-store" }) + const data = await response.json() as InferenceResponse & { error?: string } + if (!response.ok) throw new Error(data.error || "Failed to load inference configuration") + setGateway(data.gateway ?? emptyRoute()) + setSystem(data.system ?? emptyRoute()) + setProviders(Array.isArray(data.providers) ? data.providers : []) + if (data.gateway?.provider) setName(data.gateway.provider) + if (data.gateway?.model) setModel(data.gateway.model) + } catch (error) { + setMessage(error instanceof Error ? error.message : "Failed to load inference configuration") + } finally { + setLoading(false) + } + } + + useEffect(() => { + load() + }, []) + + function selectProvider(provider: ProviderSummary) { + if (!provider.name) return + setName(provider.name) + setType(provider.type || "openai") + setCredentialKey(provider.credentialKeys[0] || "OPENAI_API_KEY") + setApiKey("") + setMessage(`Loaded ${provider.name}. Enter a model and save to make it active.`) + } + + async function saveEndpoint() { + try { + setSaving(true) + setMessage("") + const response = await fetch("/api/inference", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name, + type, + model, + baseUrl, + credentialKey, + apiKey, + route, + timeout: Number(timeout || "0"), + noVerify, + makeActive: true, + }), + }) + const data = await response.json() + if (!response.ok) throw new Error(data.error || "Failed to save inference endpoint") + setGateway(data.gateway ?? gateway) + setSystem(data.system ?? system) + setApiKey("") + setMessage(`Inference endpoint '${name}' saved and routed to ${route === "system" ? "system" : "sandbox"} inference.`) + await load() + } catch (error) { + setMessage(error instanceof Error ? error.message : "Failed to save inference endpoint") + } finally { + setSaving(false) + } + } + + return ( +
+
+
+

+ INFERENCE ENDPOINTS +

+

+ Configure the OpenShell gateway route used by sandboxes through inference.local. +

+
+ +
+ + {loading ? ( +
Loading inference routes...
+ ) : ( +
+
+
+

Sandbox Route

+

{gateway.configured ? gateway.provider : "not configured"}

+

{gateway.model || "No model selected"}

+ {gateway.timeout &&

Timeout {gateway.timeout}

} +
+
+

System Route

+

{system.configured ? system.provider : "not configured"}

+

{system.model || "No model selected"}

+ {system.timeout &&

Timeout {system.timeout}

} +
+
+ +
+
+
+
+ Provider Name + setName(event.target.value)} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Provider Type + +
+
+ Model + setModel(event.target.value)} placeholder="provider/model-name" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Endpoint URL + setBaseUrl(event.target.value)} placeholder="https://example.com/v1" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Credential Env Key + setCredentialKey(event.target.value.toUpperCase())} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ API Key + setApiKey(event.target.value)} type="password" placeholder={activeProvider ? "leave blank to keep stored key" : "stored in OpenShell provider"} className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ Route + +
+
+ Timeout Seconds + setTimeoutValue(event.target.value)} inputMode="numeric" className="w-full rounded-sm border border-[var(--border-subtle)] bg-[var(--background-tertiary)] px-3 py-2 text-sm font-mono text-[var(--foreground)] focus:outline-none focus:border-[var(--nvidia-green)]" /> +
+
+ + + +
+ + {message &&

{message}

} +
+
+ +
+

Configured Providers

+ {providers.length === 0 ? ( +
No providers configured.
+ ) : ( +
+ {providers.map((provider) => ( + + ))} +
+ )} +
+
+
+ )} +
+ ) +} diff --git a/app/page.tsx b/app/page.tsx index c20c628..3ad0550 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -3,6 +3,7 @@ import { useEffect, useMemo, useState } from 'react' import Sidebar from './components/Sidebar' import SandboxList from './components/SandboxList' import ConfigurationPanel from './components/ConfigurationPanel' +import InferenceEndpointPanel from './components/InferenceEndpointPanel' import { useSandboxInventory } from './hooks/useSandboxInventory' import { createHydrationSafeDashboardSessionState, @@ -200,6 +201,7 @@ export default function Dashboard() { + ) : isCreateMode ? (
diff --git a/tests/sandbox-lifecycle-check.mjs b/tests/sandbox-lifecycle-check.mjs index 1ba49a3..205e23b 100644 --- a/tests/sandbox-lifecycle-check.mjs +++ b/tests/sandbox-lifecycle-check.mjs @@ -6,16 +6,20 @@ const root = process.cwd() const createRoutePath = path.join(root, 'app/api/sandbox/create/route.ts') const deleteRoutePath = path.join(root, 'app/api/sandbox/delete/route.ts') const telemetryRoutePath = path.join(root, 'app/api/telemetry/real/route.ts') +const inferenceRoutePath = path.join(root, 'app/api/inference/route.ts') const hookPath = path.join(root, 'app/hooks/useSandboxInventory.ts') const pagePath = path.join(root, 'app/page.tsx') const configPanelPath = path.join(root, 'app/components/ConfigurationPanel.tsx') +const inferencePanelPath = path.join(root, 'app/components/InferenceEndpointPanel.tsx') const createRouteSource = await readFile(createRoutePath, 'utf8') const deleteRouteSource = await readFile(deleteRoutePath, 'utf8') const telemetryRouteSource = await readFile(telemetryRoutePath, 'utf8') +const inferenceRouteSource = await readFile(inferenceRoutePath, 'utf8') const hookSource = await readFile(hookPath, 'utf8') const pageSource = await readFile(pagePath, 'utf8') const configPanelSource = await readFile(configPanelPath, 'utf8') +const inferencePanelSource = await readFile(inferencePanelPath, 'utf8') assert.match( createRouteSource, @@ -44,6 +48,12 @@ assert.match(deleteRouteSource, /openShellAlreadyGone/, 'delete route must toler assert.match(deleteRouteSource, /isNemoClawSandboxNotRegistered/, 'delete route must tolerate NemoClaw registry entries that were already cleaned up') assert.match(telemetryRouteSource, /const names = parseOpenShellSandboxNames\(sandboxListStdout\)/, 'telemetry must derive inventory from OpenShell before consulting NemoClaw') assert.match(telemetryRouteSource, /names\.length > 0[\s\S]*execNemoclaw\(\["list"\]\)[\s\S]*execNemoclaw\(\["status"\]\)/, 'telemetry must skip NemoClaw list/status when OpenShell reports zero live sandboxes') +assert.match(inferenceRouteSource, /\["provider", "create", "--name", name, "--type", type\]/, 'inference route must create OpenShell providers') +assert.match(inferenceRouteSource, /\["provider", "update", name\]/, 'inference route must update existing OpenShell providers') +assert.match(inferenceRouteSource, /\["inference", "set", "--provider", name, "--model", model\]/, 'inference route must set the active OpenShell inference route') +assert.match(inferenceRouteSource, /OPENAI_BASE_URL=\$\{baseUrl\}/, 'inference route must pass endpoint URL as provider config') +assert.match(inferencePanelSource, /INFERENCE ENDPOINTS/, 'settings UI must expose inference endpoint configuration') +assert.match(pageSource, //, 'settings view must render inference endpoint panel') assert.match(pageSource, /fetch\('\/api\/sandbox\/delete'/, 'destroy workflow must call the real sandbox delete endpoint') assert.match(pageSource, /refresh\(\{ force: true \}\)/, 'create and destroy workflows must force fresh inventory reads')