From 65309285fabff2cad38ce4ff6118730d391a76ca Mon Sep 17 00:00:00 2001 From: mmckeen-nv Date: Thu, 30 Apr 2026 12:41:30 -0700 Subject: [PATCH] Add live dashboard telemetry bar --- app/api/telemetry/live/route.ts | 8 + app/components/LiveTelemetryBar.tsx | 158 +++++++++++++++ app/lib/liveTelemetry.ts | 287 ++++++++++++++++++++++++++++ app/lib/mcpBrokerClient.ts | 62 ++++-- app/lib/mcpBrokerProtocol.ts | 3 + app/page.tsx | 2 + 6 files changed, 500 insertions(+), 20 deletions(-) create mode 100644 app/api/telemetry/live/route.ts create mode 100644 app/components/LiveTelemetryBar.tsx create mode 100644 app/lib/liveTelemetry.ts diff --git a/app/api/telemetry/live/route.ts b/app/api/telemetry/live/route.ts new file mode 100644 index 0000000..1af6e30 --- /dev/null +++ b/app/api/telemetry/live/route.ts @@ -0,0 +1,8 @@ +import { NextResponse } from "next/server" +import { readLiveTelemetrySnapshot } from "@/app/lib/liveTelemetry" + +export async function GET() { + return NextResponse.json(await readLiveTelemetrySnapshot(), { + headers: { "Cache-Control": "no-store" }, + }) +} diff --git a/app/components/LiveTelemetryBar.tsx b/app/components/LiveTelemetryBar.tsx new file mode 100644 index 0000000..07c101d --- /dev/null +++ b/app/components/LiveTelemetryBar.tsx @@ -0,0 +1,158 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" +import type { ReactNode } from "react" + +type LiveTelemetrySnapshot = { + ok?: boolean + timestamp?: string + windowMs?: number + transactionsPerSecond?: number + mcpRequestsPerSecond?: number + mcpCallsPerSecond?: number + tokensPerSecond?: number + promptTokensPerSecond?: number + generationTokensPerSecond?: number + tokenSource?: "vllm" | "observed" | "not_configured" | "unavailable" | string + tokenEndpoint?: string + tokenError?: string | null + concurrentTasks?: number + totals?: { + transactions?: number + mcpRequests?: number + mcpCalls?: number + mcpLists?: number + mcpErrors?: number + tokens?: number + promptTokens?: number + generationTokens?: number + } +} + +const EMPTY_SNAPSHOT: LiveTelemetrySnapshot = { + transactionsPerSecond: 0, + mcpRequestsPerSecond: 0, + mcpCallsPerSecond: 0, + tokensPerSecond: 0, + promptTokensPerSecond: 0, + generationTokensPerSecond: 0, + tokenSource: "not_configured", + concurrentTasks: 0, + totals: {}, +} + +function formatRate(value: number | undefined) { + const safeValue = Number.isFinite(value) ? Number(value) : 0 + if (safeValue >= 100) return safeValue.toFixed(0) + if (safeValue >= 10) return safeValue.toFixed(1) + return safeValue.toFixed(2) +} + +function Gauge({ value, max }: { value: number; max: number }) { + const clamped = Math.max(0, Math.min(value / Math.max(max, 1), 1)) + const angle = -120 + clamped * 240 + const dash = 126 * clamped + + return ( +
+ +
+ ) +} + +function MetricBlock({ + label, + value, + subtext, + children, +}: { + label: string + value: string + subtext: string + children?: ReactNode +}) { + return ( +
+ {children} +
+

{label}

+

{value}

+

{subtext}

+
+
+ ) +} + +export default function LiveTelemetryBar() { + const [snapshot, setSnapshot] = useState(EMPTY_SNAPSHOT) + const [error, setError] = useState("") + + useEffect(() => { + let active = true + + const load = async () => { + try { + const response = await fetch("/api/telemetry/live", { cache: "no-store" }) + const data = await response.json() + if (!response.ok) throw new Error(data.error || "Failed to load live telemetry") + if (active) { + setSnapshot(data) + setError("") + } + } catch (err) { + if (active) setError(err instanceof Error ? err.message : "Telemetry unavailable") + } + } + + load() + const interval = window.setInterval(load, 1000) + return () => { + active = false + window.clearInterval(interval) + } + }, []) + + const maxTps = useMemo(() => Math.max(5, Math.ceil((snapshot.transactionsPerSecond || 0) * 1.6)), [snapshot.transactionsPerSecond]) + const generatedRate = snapshot.generationTokensPerSecond ?? snapshot.tokensPerSecond + const promptRate = snapshot.promptTokensPerSecond || 0 + const tokenText = snapshot.tokenSource === "vllm" || snapshot.tokenSource === "observed" + ? `${formatRate(generatedRate)}/s` + : "waiting" + const tokenSubtext = snapshot.tokenSource === "vllm" + ? `generated avg; prompt ${formatRate(promptRate)}/s` + : snapshot.tokenError || "token source not wired yet" + + return ( +
+
+ + + + + +
+ {error && ( +
+ {error} +
+ )} +
+ ) +} diff --git a/app/lib/liveTelemetry.ts b/app/lib/liveTelemetry.ts new file mode 100644 index 0000000..45e7890 --- /dev/null +++ b/app/lib/liveTelemetry.ts @@ -0,0 +1,287 @@ +type TelemetryEventKind = "transaction" | "mcp_request" | "mcp_call" | "mcp_list" | "mcp_error" | "token" + +type TelemetryEvent = { + kind: TelemetryEventKind + at: number + count: number + sandboxId?: string | null + serverId?: string | null + toolName?: string | null +} + +type ActiveTask = { + id: number + kind: TelemetryEventKind + startedAt: number + sandboxId?: string | null + serverId?: string | null + toolName?: string | null +} + +type LiveTelemetryState = { + activeTasks: Map + events: TelemetryEvent[] + nextTaskId: number + tokenMetrics: { + source: string + endpoint: string + lastPromptTotal: number | null + lastGenerationTotal: number | null + lastAt: number | null + promptRate: number + generationRate: number + promptTotal: number + generationTotal: number + samples: Array<{ + at: number + promptDelta: number + generationDelta: number + elapsedSeconds: number + }> + error: string | null + } +} + +declare global { + // eslint-disable-next-line no-var + var __openshellControlLiveTelemetry: LiveTelemetryState | undefined +} + +const WINDOW_MS = 60000 + +function state() { + globalThis.__openshellControlLiveTelemetry ??= { + activeTasks: new Map(), + events: [], + nextTaskId: 1, + tokenMetrics: { + source: "not_configured", + endpoint: "", + lastPromptTotal: null, + lastGenerationTotal: null, + lastAt: null, + promptRate: 0, + generationRate: 0, + promptTotal: 0, + generationTotal: 0, + samples: [], + error: null, + }, + } + return globalThis.__openshellControlLiveTelemetry +} + +function now() { + return Date.now() +} + +function prune(cutoff = now() - WINDOW_MS) { + const { events } = state() + while (events.length > 0 && events[0].at < cutoff) events.shift() +} + +export function recordLiveTelemetryEvent( + kind: TelemetryEventKind, + options?: Omit & { count?: number }, +) { + const { events } = state() + events.push({ + kind, + at: now(), + count: Math.max(0, options?.count ?? 1), + sandboxId: options?.sandboxId ?? null, + serverId: options?.serverId ?? null, + toolName: options?.toolName ?? null, + }) + prune() +} + +export function startLiveTelemetryTask( + kind: TelemetryEventKind, + options?: Omit, +) { + const telemetryState = state() + const id = telemetryState.nextTaskId + telemetryState.nextTaskId += 1 + telemetryState.activeTasks.set(id, { + id, + kind, + startedAt: now(), + sandboxId: options?.sandboxId ?? null, + serverId: options?.serverId ?? null, + toolName: options?.toolName ?? null, + }) + return () => { + state().activeTasks.delete(id) + } +} + +function rateFor(kind: TelemetryEventKind, windowMs: number) { + const cutoff = now() - windowMs + return state().events + .filter((event) => event.kind === kind && event.at >= cutoff) + .reduce((total, event) => total + event.count, 0) / (windowMs / 1000) +} + +function countFor(kind: TelemetryEventKind, windowMs: number) { + const cutoff = now() - windowMs + return state().events + .filter((event) => event.kind === kind && event.at >= cutoff) + .reduce((total, event) => total + event.count, 0) +} + +function tokenMetricsUrl() { + const explicit = process.env.LIVE_TELEMETRY_TOKEN_METRICS_URL + || process.env.VLLM_METRICS_URL + || process.env.NEMOCLAW_VLLM_METRICS_URL + if (explicit?.trim()) return explicit.trim() + + const base = process.env.VLLM_BASE_URL + || process.env.OPENAI_BASE_URL + || process.env.MCP_PREFLIGHT_LLM_BASE_URL + if (base?.trim()) { + try { + const url = new URL(base) + url.pathname = "/metrics" + url.search = "" + url.hash = "" + return url.toString() + } catch { + return "" + } + } + + return "http://localhost:8000/metrics" +} + +function sumPrometheusMetrics(text: string, names: string[]) { + const wanted = new Set(names) + let total = 0 + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim() + if (!line || line.startsWith("#")) continue + const match = line.match(/^([A-Za-z_:][A-Za-z0-9_:]*)(?:\{[^}]*\})?\s+(-?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)$/i) + if (!match || !wanted.has(match[1])) continue + const value = Number(match[2]) + if (Number.isFinite(value)) total += value + } + return total +} + +function sumSamples(samples: LiveTelemetryState["tokenMetrics"]["samples"], field: "promptDelta" | "generationDelta") { + return samples.reduce((total, sample) => total + sample[field], 0) +} + +function sampleSeconds(samples: LiveTelemetryState["tokenMetrics"]["samples"]) { + return samples.reduce((total, sample) => total + sample.elapsedSeconds, 0) +} + +async function refreshTokenMetrics() { + const telemetryState = state() + const endpoint = tokenMetricsUrl() + if (!endpoint) { + telemetryState.tokenMetrics = { + ...telemetryState.tokenMetrics, + source: "not_configured", + endpoint: "", + promptRate: 0, + generationRate: 0, + error: "No token metrics endpoint configured", + } + return + } + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 1500) + try { + const response = await fetch(endpoint, { + cache: "no-store", + signal: controller.signal, + }) + if (!response.ok) throw new Error(`metrics returned HTTP ${response.status}`) + const metrics = await response.text() + const promptTotal = sumPrometheusMetrics(metrics, ["vllm:prompt_tokens_total"]) + const generationTotal = sumPrometheusMetrics(metrics, ["vllm:generation_tokens_total"]) + const at = now() + const previousPromptTotal = telemetryState.tokenMetrics.lastPromptTotal + const previousGenerationTotal = telemetryState.tokenMetrics.lastGenerationTotal + const previousAt = telemetryState.tokenMetrics.lastAt + const elapsedSeconds = previousAt ? (at - previousAt) / 1000 : 0 + const promptDelta = previousPromptTotal === null ? 0 : Math.max(0, promptTotal - previousPromptTotal) + const generationDelta = previousGenerationTotal === null ? 0 : Math.max(0, generationTotal - previousGenerationTotal) + const previousSamples = telemetryState.tokenMetrics.samples || [] + const samples = elapsedSeconds > 0 + ? [...previousSamples, { at, promptDelta, generationDelta, elapsedSeconds }] + : previousSamples + const cutoff = at - WINDOW_MS + const recentSamples = samples.filter((sample) => sample.at >= cutoff) + const recentSeconds = sampleSeconds(recentSamples) + + telemetryState.tokenMetrics = { + source: "vllm", + endpoint, + lastPromptTotal: promptTotal, + lastGenerationTotal: generationTotal, + lastAt: at, + promptRate: recentSeconds > 0 ? sumSamples(recentSamples, "promptDelta") / recentSeconds : 0, + generationRate: recentSeconds > 0 ? sumSamples(recentSamples, "generationDelta") / recentSeconds : 0, + promptTotal, + generationTotal, + samples: recentSamples, + error: null, + } + } catch (error) { + telemetryState.tokenMetrics = { + ...telemetryState.tokenMetrics, + source: telemetryState.tokenMetrics.lastGenerationTotal === null ? "unavailable" : telemetryState.tokenMetrics.source, + endpoint, + promptRate: 0, + generationRate: 0, + error: error instanceof Error ? error.message : "Token metrics unavailable", + } + } finally { + clearTimeout(timeout) + } +} + +export async function readLiveTelemetrySnapshot() { + await refreshTokenMetrics() + const snapshotAt = now() + prune(snapshotAt - WINDOW_MS) + const windowMs = WINDOW_MS + const active = Array.from(state().activeTasks.values()) + const tokenMetrics = state().tokenMetrics + + return { + ok: true, + timestamp: new Date(snapshotAt).toISOString(), + windowMs, + transactionsPerSecond: rateFor("transaction", windowMs), + mcpRequestsPerSecond: rateFor("mcp_request", windowMs), + mcpCallsPerSecond: rateFor("mcp_call", windowMs), + tokensPerSecond: tokenMetrics.generationRate, + promptTokensPerSecond: tokenMetrics.promptRate, + generationTokensPerSecond: tokenMetrics.generationRate, + tokenSource: tokenMetrics.source, + tokenEndpoint: tokenMetrics.endpoint, + tokenError: tokenMetrics.error, + concurrentTasks: active.length, + activeTasks: active.map((task) => ({ + kind: task.kind, + ageMs: snapshotAt - task.startedAt, + sandboxId: task.sandboxId, + serverId: task.serverId, + toolName: task.toolName, + })), + totals: { + transactions: countFor("transaction", windowMs), + mcpRequests: countFor("mcp_request", windowMs), + mcpCalls: countFor("mcp_call", windowMs), + mcpLists: countFor("mcp_list", windowMs), + mcpErrors: countFor("mcp_error", windowMs), + tokens: tokenMetrics.promptTotal + tokenMetrics.generationTotal, + promptTokens: tokenMetrics.promptTotal, + generationTokens: tokenMetrics.generationTotal, + }, + } +} diff --git a/app/lib/mcpBrokerClient.ts b/app/lib/mcpBrokerClient.ts index 5419a02..eb5d67c 100644 --- a/app/lib/mcpBrokerClient.ts +++ b/app/lib/mcpBrokerClient.ts @@ -2,6 +2,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js" import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js" import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js" import { HOST_PATH } from "./hostCommands" +import { recordLiveTelemetryEvent, startLiveTelemetryTask } from "./liveTelemetry" import type { McpServerInstall } from "./mcpServerStore" const MCP_BROKER_REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_BROKER_REQUEST_TIMEOUT_MS || "45000", 10) @@ -55,16 +56,26 @@ async function withMcpClient(server: McpServerInstall, fn: (client: Client) = } export async function listBrokerServerTools(server: McpServerInstall) { - return withMcpClient(server, async (client) => { - const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS }) - return result.tools.map((tool) => ({ - name: tool.name, - title: tool.title, - description: tool.description, - inputSchema: tool.inputSchema, - annotations: tool.annotations, - })) - }) + const stopTask = startLiveTelemetryTask("mcp_list", { serverId: server.id }) + recordLiveTelemetryEvent("transaction", { serverId: server.id }) + try { + return await withMcpClient(server, async (client) => { + const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS }) + recordLiveTelemetryEvent("mcp_list", { serverId: server.id }) + return result.tools.map((tool) => ({ + name: tool.name, + title: tool.title, + description: tool.description, + inputSchema: tool.inputSchema, + annotations: tool.annotations, + })) + }) + } catch (error) { + recordLiveTelemetryEvent("mcp_error", { serverId: server.id }) + throw error + } finally { + stopTask() + } } export async function callBrokerServerTool( @@ -72,14 +83,25 @@ export async function callBrokerServerTool( toolName: string, toolArguments: Record, ) { - return withMcpClient(server, async (client) => ( - client.callTool( - { - name: toolName, - arguments: toolArguments, - }, - undefined, - { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS }, - ) - )) + const stopTask = startLiveTelemetryTask("mcp_call", { serverId: server.id, toolName }) + recordLiveTelemetryEvent("transaction", { serverId: server.id, toolName }) + try { + const result = await withMcpClient(server, async (client) => ( + client.callTool( + { + name: toolName, + arguments: toolArguments, + }, + undefined, + { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS }, + ) + )) + recordLiveTelemetryEvent("mcp_call", { serverId: server.id, toolName }) + return result + } catch (error) { + recordLiveTelemetryEvent("mcp_error", { serverId: server.id, toolName }) + throw error + } finally { + stopTask() + } } diff --git a/app/lib/mcpBrokerProtocol.ts b/app/lib/mcpBrokerProtocol.ts index c71ae47..a0a80d2 100644 --- a/app/lib/mcpBrokerProtocol.ts +++ b/app/lib/mcpBrokerProtocol.ts @@ -1,4 +1,5 @@ import { callBrokerServerTool, listBrokerServerTools } from "./mcpBrokerClient" +import { recordLiveTelemetryEvent } from "./liveTelemetry" import { listAllowedBrokerServers, type SandboxMcpBrokerSession } from "./mcpBrokerStore" import type { McpServerInstall } from "./mcpServerStore" @@ -114,6 +115,8 @@ async function handleBrokerMcpJsonRpcMessage(session: SandboxMcpBrokerSession, m const id = hasJsonRpcId(request) ? request.id ?? null : null const method = typeof request.method === "string" ? request.method : "" const isNotification = !hasJsonRpcId(request) + recordLiveTelemetryEvent("transaction", { sandboxId: session.sandboxId }) + recordLiveTelemetryEvent("mcp_request", { sandboxId: session.sandboxId }) try { switch (method) { diff --git a/app/page.tsx b/app/page.tsx index 3562182..3960ebf 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -7,6 +7,7 @@ import InferenceEndpointPanel from './components/InferenceEndpointPanel' import HelpPanel from './components/HelpPanel' import WizardPanel from './components/WizardPanel' import McpConfigurationPanel from './components/McpConfigurationPanel' +import LiveTelemetryBar from './components/LiveTelemetryBar' import { useSandboxInventory } from './hooks/useSandboxInventory' import { createHydrationSafeDashboardSessionState, @@ -237,6 +238,7 @@ export default function Dashboard() {
+ {activeView === 'settings' ? (