Add live dashboard telemetry bar

This commit is contained in:
mmckeen-nv
2026-04-30 12:41:30 -07:00
parent 6d25b2395e
commit 65309285fa
6 changed files with 500 additions and 20 deletions
+287
View File
@@ -0,0 +1,287 @@
type TelemetryEventKind = "transaction" | "mcp_request" | "mcp_call" | "mcp_list" | "mcp_error" | "token"
type TelemetryEvent = {
kind: TelemetryEventKind
at: number
count: number
sandboxId?: string | null
serverId?: string | null
toolName?: string | null
}
type ActiveTask = {
id: number
kind: TelemetryEventKind
startedAt: number
sandboxId?: string | null
serverId?: string | null
toolName?: string | null
}
type LiveTelemetryState = {
activeTasks: Map<number, ActiveTask>
events: TelemetryEvent[]
nextTaskId: number
tokenMetrics: {
source: string
endpoint: string
lastPromptTotal: number | null
lastGenerationTotal: number | null
lastAt: number | null
promptRate: number
generationRate: number
promptTotal: number
generationTotal: number
samples: Array<{
at: number
promptDelta: number
generationDelta: number
elapsedSeconds: number
}>
error: string | null
}
}
declare global {
// eslint-disable-next-line no-var
var __openshellControlLiveTelemetry: LiveTelemetryState | undefined
}
const WINDOW_MS = 60000
function state() {
globalThis.__openshellControlLiveTelemetry ??= {
activeTasks: new Map<number, ActiveTask>(),
events: [],
nextTaskId: 1,
tokenMetrics: {
source: "not_configured",
endpoint: "",
lastPromptTotal: null,
lastGenerationTotal: null,
lastAt: null,
promptRate: 0,
generationRate: 0,
promptTotal: 0,
generationTotal: 0,
samples: [],
error: null,
},
}
return globalThis.__openshellControlLiveTelemetry
}
function now() {
return Date.now()
}
function prune(cutoff = now() - WINDOW_MS) {
const { events } = state()
while (events.length > 0 && events[0].at < cutoff) events.shift()
}
export function recordLiveTelemetryEvent(
kind: TelemetryEventKind,
options?: Omit<TelemetryEvent, "kind" | "at" | "count"> & { count?: number },
) {
const { events } = state()
events.push({
kind,
at: now(),
count: Math.max(0, options?.count ?? 1),
sandboxId: options?.sandboxId ?? null,
serverId: options?.serverId ?? null,
toolName: options?.toolName ?? null,
})
prune()
}
export function startLiveTelemetryTask(
kind: TelemetryEventKind,
options?: Omit<ActiveTask, "id" | "kind" | "startedAt">,
) {
const telemetryState = state()
const id = telemetryState.nextTaskId
telemetryState.nextTaskId += 1
telemetryState.activeTasks.set(id, {
id,
kind,
startedAt: now(),
sandboxId: options?.sandboxId ?? null,
serverId: options?.serverId ?? null,
toolName: options?.toolName ?? null,
})
return () => {
state().activeTasks.delete(id)
}
}
function rateFor(kind: TelemetryEventKind, windowMs: number) {
const cutoff = now() - windowMs
return state().events
.filter((event) => event.kind === kind && event.at >= cutoff)
.reduce((total, event) => total + event.count, 0) / (windowMs / 1000)
}
function countFor(kind: TelemetryEventKind, windowMs: number) {
const cutoff = now() - windowMs
return state().events
.filter((event) => event.kind === kind && event.at >= cutoff)
.reduce((total, event) => total + event.count, 0)
}
function tokenMetricsUrl() {
const explicit = process.env.LIVE_TELEMETRY_TOKEN_METRICS_URL
|| process.env.VLLM_METRICS_URL
|| process.env.NEMOCLAW_VLLM_METRICS_URL
if (explicit?.trim()) return explicit.trim()
const base = process.env.VLLM_BASE_URL
|| process.env.OPENAI_BASE_URL
|| process.env.MCP_PREFLIGHT_LLM_BASE_URL
if (base?.trim()) {
try {
const url = new URL(base)
url.pathname = "/metrics"
url.search = ""
url.hash = ""
return url.toString()
} catch {
return ""
}
}
return "http://localhost:8000/metrics"
}
function sumPrometheusMetrics(text: string, names: string[]) {
const wanted = new Set(names)
let total = 0
for (const rawLine of text.split(/\r?\n/)) {
const line = rawLine.trim()
if (!line || line.startsWith("#")) continue
const match = line.match(/^([A-Za-z_:][A-Za-z0-9_:]*)(?:\{[^}]*\})?\s+(-?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)$/i)
if (!match || !wanted.has(match[1])) continue
const value = Number(match[2])
if (Number.isFinite(value)) total += value
}
return total
}
function sumSamples(samples: LiveTelemetryState["tokenMetrics"]["samples"], field: "promptDelta" | "generationDelta") {
return samples.reduce((total, sample) => total + sample[field], 0)
}
function sampleSeconds(samples: LiveTelemetryState["tokenMetrics"]["samples"]) {
return samples.reduce((total, sample) => total + sample.elapsedSeconds, 0)
}
async function refreshTokenMetrics() {
const telemetryState = state()
const endpoint = tokenMetricsUrl()
if (!endpoint) {
telemetryState.tokenMetrics = {
...telemetryState.tokenMetrics,
source: "not_configured",
endpoint: "",
promptRate: 0,
generationRate: 0,
error: "No token metrics endpoint configured",
}
return
}
const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), 1500)
try {
const response = await fetch(endpoint, {
cache: "no-store",
signal: controller.signal,
})
if (!response.ok) throw new Error(`metrics returned HTTP ${response.status}`)
const metrics = await response.text()
const promptTotal = sumPrometheusMetrics(metrics, ["vllm:prompt_tokens_total"])
const generationTotal = sumPrometheusMetrics(metrics, ["vllm:generation_tokens_total"])
const at = now()
const previousPromptTotal = telemetryState.tokenMetrics.lastPromptTotal
const previousGenerationTotal = telemetryState.tokenMetrics.lastGenerationTotal
const previousAt = telemetryState.tokenMetrics.lastAt
const elapsedSeconds = previousAt ? (at - previousAt) / 1000 : 0
const promptDelta = previousPromptTotal === null ? 0 : Math.max(0, promptTotal - previousPromptTotal)
const generationDelta = previousGenerationTotal === null ? 0 : Math.max(0, generationTotal - previousGenerationTotal)
const previousSamples = telemetryState.tokenMetrics.samples || []
const samples = elapsedSeconds > 0
? [...previousSamples, { at, promptDelta, generationDelta, elapsedSeconds }]
: previousSamples
const cutoff = at - WINDOW_MS
const recentSamples = samples.filter((sample) => sample.at >= cutoff)
const recentSeconds = sampleSeconds(recentSamples)
telemetryState.tokenMetrics = {
source: "vllm",
endpoint,
lastPromptTotal: promptTotal,
lastGenerationTotal: generationTotal,
lastAt: at,
promptRate: recentSeconds > 0 ? sumSamples(recentSamples, "promptDelta") / recentSeconds : 0,
generationRate: recentSeconds > 0 ? sumSamples(recentSamples, "generationDelta") / recentSeconds : 0,
promptTotal,
generationTotal,
samples: recentSamples,
error: null,
}
} catch (error) {
telemetryState.tokenMetrics = {
...telemetryState.tokenMetrics,
source: telemetryState.tokenMetrics.lastGenerationTotal === null ? "unavailable" : telemetryState.tokenMetrics.source,
endpoint,
promptRate: 0,
generationRate: 0,
error: error instanceof Error ? error.message : "Token metrics unavailable",
}
} finally {
clearTimeout(timeout)
}
}
export async function readLiveTelemetrySnapshot() {
await refreshTokenMetrics()
const snapshotAt = now()
prune(snapshotAt - WINDOW_MS)
const windowMs = WINDOW_MS
const active = Array.from(state().activeTasks.values())
const tokenMetrics = state().tokenMetrics
return {
ok: true,
timestamp: new Date(snapshotAt).toISOString(),
windowMs,
transactionsPerSecond: rateFor("transaction", windowMs),
mcpRequestsPerSecond: rateFor("mcp_request", windowMs),
mcpCallsPerSecond: rateFor("mcp_call", windowMs),
tokensPerSecond: tokenMetrics.generationRate,
promptTokensPerSecond: tokenMetrics.promptRate,
generationTokensPerSecond: tokenMetrics.generationRate,
tokenSource: tokenMetrics.source,
tokenEndpoint: tokenMetrics.endpoint,
tokenError: tokenMetrics.error,
concurrentTasks: active.length,
activeTasks: active.map((task) => ({
kind: task.kind,
ageMs: snapshotAt - task.startedAt,
sandboxId: task.sandboxId,
serverId: task.serverId,
toolName: task.toolName,
})),
totals: {
transactions: countFor("transaction", windowMs),
mcpRequests: countFor("mcp_request", windowMs),
mcpCalls: countFor("mcp_call", windowMs),
mcpLists: countFor("mcp_list", windowMs),
mcpErrors: countFor("mcp_error", windowMs),
tokens: tokenMetrics.promptTotal + tokenMetrics.generationTotal,
promptTokens: tokenMetrics.promptTotal,
generationTokens: tokenMetrics.generationTotal,
},
}
}
+42 -20
View File
@@ -2,6 +2,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"
import { HOST_PATH } from "./hostCommands"
import { recordLiveTelemetryEvent, startLiveTelemetryTask } from "./liveTelemetry"
import type { McpServerInstall } from "./mcpServerStore"
const MCP_BROKER_REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_BROKER_REQUEST_TIMEOUT_MS || "45000", 10)
@@ -55,16 +56,26 @@ async function withMcpClient<T>(server: McpServerInstall, fn: (client: Client) =
}
export async function listBrokerServerTools(server: McpServerInstall) {
return withMcpClient(server, async (client) => {
const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS })
return result.tools.map((tool) => ({
name: tool.name,
title: tool.title,
description: tool.description,
inputSchema: tool.inputSchema,
annotations: tool.annotations,
}))
})
const stopTask = startLiveTelemetryTask("mcp_list", { serverId: server.id })
recordLiveTelemetryEvent("transaction", { serverId: server.id })
try {
return await withMcpClient(server, async (client) => {
const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS })
recordLiveTelemetryEvent("mcp_list", { serverId: server.id })
return result.tools.map((tool) => ({
name: tool.name,
title: tool.title,
description: tool.description,
inputSchema: tool.inputSchema,
annotations: tool.annotations,
}))
})
} catch (error) {
recordLiveTelemetryEvent("mcp_error", { serverId: server.id })
throw error
} finally {
stopTask()
}
}
export async function callBrokerServerTool(
@@ -72,14 +83,25 @@ export async function callBrokerServerTool(
toolName: string,
toolArguments: Record<string, unknown>,
) {
return withMcpClient(server, async (client) => (
client.callTool(
{
name: toolName,
arguments: toolArguments,
},
undefined,
{ timeout: MCP_BROKER_REQUEST_TIMEOUT_MS },
)
))
const stopTask = startLiveTelemetryTask("mcp_call", { serverId: server.id, toolName })
recordLiveTelemetryEvent("transaction", { serverId: server.id, toolName })
try {
const result = await withMcpClient(server, async (client) => (
client.callTool(
{
name: toolName,
arguments: toolArguments,
},
undefined,
{ timeout: MCP_BROKER_REQUEST_TIMEOUT_MS },
)
))
recordLiveTelemetryEvent("mcp_call", { serverId: server.id, toolName })
return result
} catch (error) {
recordLiveTelemetryEvent("mcp_error", { serverId: server.id, toolName })
throw error
} finally {
stopTask()
}
}
+3
View File
@@ -1,4 +1,5 @@
import { callBrokerServerTool, listBrokerServerTools } from "./mcpBrokerClient"
import { recordLiveTelemetryEvent } from "./liveTelemetry"
import { listAllowedBrokerServers, type SandboxMcpBrokerSession } from "./mcpBrokerStore"
import type { McpServerInstall } from "./mcpServerStore"
@@ -114,6 +115,8 @@ async function handleBrokerMcpJsonRpcMessage(session: SandboxMcpBrokerSession, m
const id = hasJsonRpcId(request) ? request.id ?? null : null
const method = typeof request.method === "string" ? request.method : ""
const isNotification = !hasJsonRpcId(request)
recordLiveTelemetryEvent("transaction", { sandboxId: session.sandboxId })
recordLiveTelemetryEvent("mcp_request", { sandboxId: session.sandboxId })
try {
switch (method) {