Add live dashboard telemetry bar

This commit is contained in:
mmckeen-nv
2026-04-30 12:41:30 -07:00
parent 6d25b2395e
commit 65309285fa
6 changed files with 500 additions and 20 deletions
+8
View File
@@ -0,0 +1,8 @@
import { NextResponse } from "next/server"
import { readLiveTelemetrySnapshot } from "@/app/lib/liveTelemetry"
export async function GET() {
return NextResponse.json(await readLiveTelemetrySnapshot(), {
headers: { "Cache-Control": "no-store" },
})
}
+158
View File
@@ -0,0 +1,158 @@
"use client"
import { useEffect, useMemo, useState } from "react"
import type { ReactNode } from "react"
type LiveTelemetrySnapshot = {
ok?: boolean
timestamp?: string
windowMs?: number
transactionsPerSecond?: number
mcpRequestsPerSecond?: number
mcpCallsPerSecond?: number
tokensPerSecond?: number
promptTokensPerSecond?: number
generationTokensPerSecond?: number
tokenSource?: "vllm" | "observed" | "not_configured" | "unavailable" | string
tokenEndpoint?: string
tokenError?: string | null
concurrentTasks?: number
totals?: {
transactions?: number
mcpRequests?: number
mcpCalls?: number
mcpLists?: number
mcpErrors?: number
tokens?: number
promptTokens?: number
generationTokens?: number
}
}
const EMPTY_SNAPSHOT: LiveTelemetrySnapshot = {
transactionsPerSecond: 0,
mcpRequestsPerSecond: 0,
mcpCallsPerSecond: 0,
tokensPerSecond: 0,
promptTokensPerSecond: 0,
generationTokensPerSecond: 0,
tokenSource: "not_configured",
concurrentTasks: 0,
totals: {},
}
function formatRate(value: number | undefined) {
const safeValue = Number.isFinite(value) ? Number(value) : 0
if (safeValue >= 100) return safeValue.toFixed(0)
if (safeValue >= 10) return safeValue.toFixed(1)
return safeValue.toFixed(2)
}
function Gauge({ value, max }: { value: number; max: number }) {
const clamped = Math.max(0, Math.min(value / Math.max(max, 1), 1))
const angle = -120 + clamped * 240
const dash = 126 * clamped
return (
<div className="relative h-16 w-20 shrink-0">
<svg viewBox="0 0 80 56" className="h-full w-full" aria-hidden="true">
<path d="M12 46a28 28 0 0 1 56 0" fill="none" stroke="var(--border-subtle)" strokeWidth="8" strokeLinecap="square" pathLength="126" />
<path d="M12 46a28 28 0 0 1 56 0" fill="none" stroke="var(--nvidia-green)" strokeWidth="8" strokeLinecap="square" pathLength="126" strokeDasharray={`${dash} 126`} />
<line x1="40" y1="46" x2="40" y2="22" stroke="var(--foreground)" strokeWidth="2" strokeLinecap="square" transform={`rotate(${angle} 40 46)`} />
<circle cx="40" cy="46" r="3" fill="var(--foreground)" />
</svg>
</div>
)
}
function MetricBlock({
label,
value,
subtext,
children,
}: {
label: string
value: string
subtext: string
children?: ReactNode
}) {
return (
<div className="flex min-w-0 items-center gap-4 border-l border-[var(--border-subtle)] px-5 first:border-l-0 max-md:border-l-0 max-md:border-t max-md:py-4">
{children}
<div className="min-w-0">
<p className="text-[10px] uppercase tracking-wider text-[var(--foreground-dim)]">{label}</p>
<p className="mt-1 font-mono text-2xl text-[var(--foreground)]">{value}</p>
<p className="mt-1 truncate text-[11px] text-[var(--foreground-dim)]">{subtext}</p>
</div>
</div>
)
}
export default function LiveTelemetryBar() {
const [snapshot, setSnapshot] = useState<LiveTelemetrySnapshot>(EMPTY_SNAPSHOT)
const [error, setError] = useState("")
useEffect(() => {
let active = true
const load = async () => {
try {
const response = await fetch("/api/telemetry/live", { cache: "no-store" })
const data = await response.json()
if (!response.ok) throw new Error(data.error || "Failed to load live telemetry")
if (active) {
setSnapshot(data)
setError("")
}
} catch (err) {
if (active) setError(err instanceof Error ? err.message : "Telemetry unavailable")
}
}
load()
const interval = window.setInterval(load, 1000)
return () => {
active = false
window.clearInterval(interval)
}
}, [])
const maxTps = useMemo(() => Math.max(5, Math.ceil((snapshot.transactionsPerSecond || 0) * 1.6)), [snapshot.transactionsPerSecond])
const generatedRate = snapshot.generationTokensPerSecond ?? snapshot.tokensPerSecond
const promptRate = snapshot.promptTokensPerSecond || 0
const tokenText = snapshot.tokenSource === "vllm" || snapshot.tokenSource === "observed"
? `${formatRate(generatedRate)}/s`
: "waiting"
const tokenSubtext = snapshot.tokenSource === "vllm"
? `generated avg; prompt ${formatRate(promptRate)}/s`
: snapshot.tokenError || "token source not wired yet"
return (
<section className="panel mb-6 overflow-hidden" aria-label="Live activity telemetry">
<div className="grid grid-cols-1 md:grid-cols-[1.1fr_1fr_1fr]">
<MetricBlock
label="Transactions Per Second"
value={`${formatRate(snapshot.transactionsPerSecond)}/s`}
subtext={`MCP requests ${formatRate(snapshot.mcpRequestsPerSecond)}/s over ${Math.round((snapshot.windowMs || 10000) / 1000)}s`}
>
<Gauge value={snapshot.transactionsPerSecond || 0} max={maxTps} />
</MetricBlock>
<MetricBlock
label="Generated Tokens Per Second"
value={tokenText}
subtext={tokenSubtext}
/>
<MetricBlock
label="Concurrent Tasks"
value={String(snapshot.concurrentTasks || 0)}
subtext={`${snapshot.totals?.mcpRequests || 0} MCP requests, ${snapshot.totals?.mcpCalls || 0} tool calls`}
/>
</div>
{error && (
<div className="border-t border-[var(--border-subtle)] px-5 py-2 text-[11px] text-[var(--status-stopped)]">
{error}
</div>
)}
</section>
)
}
+287
View File
@@ -0,0 +1,287 @@
type TelemetryEventKind = "transaction" | "mcp_request" | "mcp_call" | "mcp_list" | "mcp_error" | "token"
type TelemetryEvent = {
kind: TelemetryEventKind
at: number
count: number
sandboxId?: string | null
serverId?: string | null
toolName?: string | null
}
type ActiveTask = {
id: number
kind: TelemetryEventKind
startedAt: number
sandboxId?: string | null
serverId?: string | null
toolName?: string | null
}
type LiveTelemetryState = {
activeTasks: Map<number, ActiveTask>
events: TelemetryEvent[]
nextTaskId: number
tokenMetrics: {
source: string
endpoint: string
lastPromptTotal: number | null
lastGenerationTotal: number | null
lastAt: number | null
promptRate: number
generationRate: number
promptTotal: number
generationTotal: number
samples: Array<{
at: number
promptDelta: number
generationDelta: number
elapsedSeconds: number
}>
error: string | null
}
}
declare global {
// eslint-disable-next-line no-var
var __openshellControlLiveTelemetry: LiveTelemetryState | undefined
}
const WINDOW_MS = 60000
function state() {
globalThis.__openshellControlLiveTelemetry ??= {
activeTasks: new Map<number, ActiveTask>(),
events: [],
nextTaskId: 1,
tokenMetrics: {
source: "not_configured",
endpoint: "",
lastPromptTotal: null,
lastGenerationTotal: null,
lastAt: null,
promptRate: 0,
generationRate: 0,
promptTotal: 0,
generationTotal: 0,
samples: [],
error: null,
},
}
return globalThis.__openshellControlLiveTelemetry
}
function now() {
return Date.now()
}
function prune(cutoff = now() - WINDOW_MS) {
const { events } = state()
while (events.length > 0 && events[0].at < cutoff) events.shift()
}
export function recordLiveTelemetryEvent(
kind: TelemetryEventKind,
options?: Omit<TelemetryEvent, "kind" | "at" | "count"> & { count?: number },
) {
const { events } = state()
events.push({
kind,
at: now(),
count: Math.max(0, options?.count ?? 1),
sandboxId: options?.sandboxId ?? null,
serverId: options?.serverId ?? null,
toolName: options?.toolName ?? null,
})
prune()
}
export function startLiveTelemetryTask(
kind: TelemetryEventKind,
options?: Omit<ActiveTask, "id" | "kind" | "startedAt">,
) {
const telemetryState = state()
const id = telemetryState.nextTaskId
telemetryState.nextTaskId += 1
telemetryState.activeTasks.set(id, {
id,
kind,
startedAt: now(),
sandboxId: options?.sandboxId ?? null,
serverId: options?.serverId ?? null,
toolName: options?.toolName ?? null,
})
return () => {
state().activeTasks.delete(id)
}
}
function rateFor(kind: TelemetryEventKind, windowMs: number) {
const cutoff = now() - windowMs
return state().events
.filter((event) => event.kind === kind && event.at >= cutoff)
.reduce((total, event) => total + event.count, 0) / (windowMs / 1000)
}
function countFor(kind: TelemetryEventKind, windowMs: number) {
const cutoff = now() - windowMs
return state().events
.filter((event) => event.kind === kind && event.at >= cutoff)
.reduce((total, event) => total + event.count, 0)
}
function tokenMetricsUrl() {
const explicit = process.env.LIVE_TELEMETRY_TOKEN_METRICS_URL
|| process.env.VLLM_METRICS_URL
|| process.env.NEMOCLAW_VLLM_METRICS_URL
if (explicit?.trim()) return explicit.trim()
const base = process.env.VLLM_BASE_URL
|| process.env.OPENAI_BASE_URL
|| process.env.MCP_PREFLIGHT_LLM_BASE_URL
if (base?.trim()) {
try {
const url = new URL(base)
url.pathname = "/metrics"
url.search = ""
url.hash = ""
return url.toString()
} catch {
return ""
}
}
return "http://localhost:8000/metrics"
}
function sumPrometheusMetrics(text: string, names: string[]) {
const wanted = new Set(names)
let total = 0
for (const rawLine of text.split(/\r?\n/)) {
const line = rawLine.trim()
if (!line || line.startsWith("#")) continue
const match = line.match(/^([A-Za-z_:][A-Za-z0-9_:]*)(?:\{[^}]*\})?\s+(-?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)$/i)
if (!match || !wanted.has(match[1])) continue
const value = Number(match[2])
if (Number.isFinite(value)) total += value
}
return total
}
function sumSamples(samples: LiveTelemetryState["tokenMetrics"]["samples"], field: "promptDelta" | "generationDelta") {
return samples.reduce((total, sample) => total + sample[field], 0)
}
function sampleSeconds(samples: LiveTelemetryState["tokenMetrics"]["samples"]) {
return samples.reduce((total, sample) => total + sample.elapsedSeconds, 0)
}
async function refreshTokenMetrics() {
const telemetryState = state()
const endpoint = tokenMetricsUrl()
if (!endpoint) {
telemetryState.tokenMetrics = {
...telemetryState.tokenMetrics,
source: "not_configured",
endpoint: "",
promptRate: 0,
generationRate: 0,
error: "No token metrics endpoint configured",
}
return
}
const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), 1500)
try {
const response = await fetch(endpoint, {
cache: "no-store",
signal: controller.signal,
})
if (!response.ok) throw new Error(`metrics returned HTTP ${response.status}`)
const metrics = await response.text()
const promptTotal = sumPrometheusMetrics(metrics, ["vllm:prompt_tokens_total"])
const generationTotal = sumPrometheusMetrics(metrics, ["vllm:generation_tokens_total"])
const at = now()
const previousPromptTotal = telemetryState.tokenMetrics.lastPromptTotal
const previousGenerationTotal = telemetryState.tokenMetrics.lastGenerationTotal
const previousAt = telemetryState.tokenMetrics.lastAt
const elapsedSeconds = previousAt ? (at - previousAt) / 1000 : 0
const promptDelta = previousPromptTotal === null ? 0 : Math.max(0, promptTotal - previousPromptTotal)
const generationDelta = previousGenerationTotal === null ? 0 : Math.max(0, generationTotal - previousGenerationTotal)
const previousSamples = telemetryState.tokenMetrics.samples || []
const samples = elapsedSeconds > 0
? [...previousSamples, { at, promptDelta, generationDelta, elapsedSeconds }]
: previousSamples
const cutoff = at - WINDOW_MS
const recentSamples = samples.filter((sample) => sample.at >= cutoff)
const recentSeconds = sampleSeconds(recentSamples)
telemetryState.tokenMetrics = {
source: "vllm",
endpoint,
lastPromptTotal: promptTotal,
lastGenerationTotal: generationTotal,
lastAt: at,
promptRate: recentSeconds > 0 ? sumSamples(recentSamples, "promptDelta") / recentSeconds : 0,
generationRate: recentSeconds > 0 ? sumSamples(recentSamples, "generationDelta") / recentSeconds : 0,
promptTotal,
generationTotal,
samples: recentSamples,
error: null,
}
} catch (error) {
telemetryState.tokenMetrics = {
...telemetryState.tokenMetrics,
source: telemetryState.tokenMetrics.lastGenerationTotal === null ? "unavailable" : telemetryState.tokenMetrics.source,
endpoint,
promptRate: 0,
generationRate: 0,
error: error instanceof Error ? error.message : "Token metrics unavailable",
}
} finally {
clearTimeout(timeout)
}
}
export async function readLiveTelemetrySnapshot() {
await refreshTokenMetrics()
const snapshotAt = now()
prune(snapshotAt - WINDOW_MS)
const windowMs = WINDOW_MS
const active = Array.from(state().activeTasks.values())
const tokenMetrics = state().tokenMetrics
return {
ok: true,
timestamp: new Date(snapshotAt).toISOString(),
windowMs,
transactionsPerSecond: rateFor("transaction", windowMs),
mcpRequestsPerSecond: rateFor("mcp_request", windowMs),
mcpCallsPerSecond: rateFor("mcp_call", windowMs),
tokensPerSecond: tokenMetrics.generationRate,
promptTokensPerSecond: tokenMetrics.promptRate,
generationTokensPerSecond: tokenMetrics.generationRate,
tokenSource: tokenMetrics.source,
tokenEndpoint: tokenMetrics.endpoint,
tokenError: tokenMetrics.error,
concurrentTasks: active.length,
activeTasks: active.map((task) => ({
kind: task.kind,
ageMs: snapshotAt - task.startedAt,
sandboxId: task.sandboxId,
serverId: task.serverId,
toolName: task.toolName,
})),
totals: {
transactions: countFor("transaction", windowMs),
mcpRequests: countFor("mcp_request", windowMs),
mcpCalls: countFor("mcp_call", windowMs),
mcpLists: countFor("mcp_list", windowMs),
mcpErrors: countFor("mcp_error", windowMs),
tokens: tokenMetrics.promptTotal + tokenMetrics.generationTotal,
promptTokens: tokenMetrics.promptTotal,
generationTokens: tokenMetrics.generationTotal,
},
}
}
+24 -2
View File
@@ -2,6 +2,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js" import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js" import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"
import { HOST_PATH } from "./hostCommands" import { HOST_PATH } from "./hostCommands"
import { recordLiveTelemetryEvent, startLiveTelemetryTask } from "./liveTelemetry"
import type { McpServerInstall } from "./mcpServerStore" import type { McpServerInstall } from "./mcpServerStore"
const MCP_BROKER_REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_BROKER_REQUEST_TIMEOUT_MS || "45000", 10) const MCP_BROKER_REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_BROKER_REQUEST_TIMEOUT_MS || "45000", 10)
@@ -55,8 +56,12 @@ async function withMcpClient<T>(server: McpServerInstall, fn: (client: Client) =
} }
export async function listBrokerServerTools(server: McpServerInstall) { export async function listBrokerServerTools(server: McpServerInstall) {
return withMcpClient(server, async (client) => { const stopTask = startLiveTelemetryTask("mcp_list", { serverId: server.id })
recordLiveTelemetryEvent("transaction", { serverId: server.id })
try {
return await withMcpClient(server, async (client) => {
const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS }) const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS })
recordLiveTelemetryEvent("mcp_list", { serverId: server.id })
return result.tools.map((tool) => ({ return result.tools.map((tool) => ({
name: tool.name, name: tool.name,
title: tool.title, title: tool.title,
@@ -65,6 +70,12 @@ export async function listBrokerServerTools(server: McpServerInstall) {
annotations: tool.annotations, annotations: tool.annotations,
})) }))
}) })
} catch (error) {
recordLiveTelemetryEvent("mcp_error", { serverId: server.id })
throw error
} finally {
stopTask()
}
} }
export async function callBrokerServerTool( export async function callBrokerServerTool(
@@ -72,7 +83,10 @@ export async function callBrokerServerTool(
toolName: string, toolName: string,
toolArguments: Record<string, unknown>, toolArguments: Record<string, unknown>,
) { ) {
return withMcpClient(server, async (client) => ( const stopTask = startLiveTelemetryTask("mcp_call", { serverId: server.id, toolName })
recordLiveTelemetryEvent("transaction", { serverId: server.id, toolName })
try {
const result = await withMcpClient(server, async (client) => (
client.callTool( client.callTool(
{ {
name: toolName, name: toolName,
@@ -82,4 +96,12 @@ export async function callBrokerServerTool(
{ timeout: MCP_BROKER_REQUEST_TIMEOUT_MS }, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS },
) )
)) ))
recordLiveTelemetryEvent("mcp_call", { serverId: server.id, toolName })
return result
} catch (error) {
recordLiveTelemetryEvent("mcp_error", { serverId: server.id, toolName })
throw error
} finally {
stopTask()
}
} }
+3
View File
@@ -1,4 +1,5 @@
import { callBrokerServerTool, listBrokerServerTools } from "./mcpBrokerClient" import { callBrokerServerTool, listBrokerServerTools } from "./mcpBrokerClient"
import { recordLiveTelemetryEvent } from "./liveTelemetry"
import { listAllowedBrokerServers, type SandboxMcpBrokerSession } from "./mcpBrokerStore" import { listAllowedBrokerServers, type SandboxMcpBrokerSession } from "./mcpBrokerStore"
import type { McpServerInstall } from "./mcpServerStore" import type { McpServerInstall } from "./mcpServerStore"
@@ -114,6 +115,8 @@ async function handleBrokerMcpJsonRpcMessage(session: SandboxMcpBrokerSession, m
const id = hasJsonRpcId(request) ? request.id ?? null : null const id = hasJsonRpcId(request) ? request.id ?? null : null
const method = typeof request.method === "string" ? request.method : "" const method = typeof request.method === "string" ? request.method : ""
const isNotification = !hasJsonRpcId(request) const isNotification = !hasJsonRpcId(request)
recordLiveTelemetryEvent("transaction", { sandboxId: session.sandboxId })
recordLiveTelemetryEvent("mcp_request", { sandboxId: session.sandboxId })
try { try {
switch (method) { switch (method) {
+2
View File
@@ -7,6 +7,7 @@ import InferenceEndpointPanel from './components/InferenceEndpointPanel'
import HelpPanel from './components/HelpPanel' import HelpPanel from './components/HelpPanel'
import WizardPanel from './components/WizardPanel' import WizardPanel from './components/WizardPanel'
import McpConfigurationPanel from './components/McpConfigurationPanel' import McpConfigurationPanel from './components/McpConfigurationPanel'
import LiveTelemetryBar from './components/LiveTelemetryBar'
import { useSandboxInventory } from './hooks/useSandboxInventory' import { useSandboxInventory } from './hooks/useSandboxInventory'
import { import {
createHydrationSafeDashboardSessionState, createHydrationSafeDashboardSessionState,
@@ -237,6 +238,7 @@ export default function Dashboard() {
<main className="ml-64 min-h-screen transition-all duration-300 max-lg:ml-0 max-lg:pb-28"> <main className="ml-64 min-h-screen transition-all duration-300 max-lg:ml-0 max-lg:pb-28">
<div> <div>
<div className="mx-auto max-w-7xl p-8 max-sm:p-4"> <div className="mx-auto max-w-7xl p-8 max-sm:p-4">
<LiveTelemetryBar />
{activeView === 'settings' ? ( {activeView === 'settings' ? (
<div className="space-y-6"> <div className="space-y-6">
<div className="panel p-8"> <div className="panel p-8">