Add live dashboard telemetry bar
This commit is contained in:
@@ -0,0 +1,287 @@
|
||||
type TelemetryEventKind = "transaction" | "mcp_request" | "mcp_call" | "mcp_list" | "mcp_error" | "token"
|
||||
|
||||
type TelemetryEvent = {
|
||||
kind: TelemetryEventKind
|
||||
at: number
|
||||
count: number
|
||||
sandboxId?: string | null
|
||||
serverId?: string | null
|
||||
toolName?: string | null
|
||||
}
|
||||
|
||||
type ActiveTask = {
|
||||
id: number
|
||||
kind: TelemetryEventKind
|
||||
startedAt: number
|
||||
sandboxId?: string | null
|
||||
serverId?: string | null
|
||||
toolName?: string | null
|
||||
}
|
||||
|
||||
type LiveTelemetryState = {
|
||||
activeTasks: Map<number, ActiveTask>
|
||||
events: TelemetryEvent[]
|
||||
nextTaskId: number
|
||||
tokenMetrics: {
|
||||
source: string
|
||||
endpoint: string
|
||||
lastPromptTotal: number | null
|
||||
lastGenerationTotal: number | null
|
||||
lastAt: number | null
|
||||
promptRate: number
|
||||
generationRate: number
|
||||
promptTotal: number
|
||||
generationTotal: number
|
||||
samples: Array<{
|
||||
at: number
|
||||
promptDelta: number
|
||||
generationDelta: number
|
||||
elapsedSeconds: number
|
||||
}>
|
||||
error: string | null
|
||||
}
|
||||
}
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var __openshellControlLiveTelemetry: LiveTelemetryState | undefined
|
||||
}
|
||||
|
||||
const WINDOW_MS = 60000
|
||||
|
||||
function state() {
|
||||
globalThis.__openshellControlLiveTelemetry ??= {
|
||||
activeTasks: new Map<number, ActiveTask>(),
|
||||
events: [],
|
||||
nextTaskId: 1,
|
||||
tokenMetrics: {
|
||||
source: "not_configured",
|
||||
endpoint: "",
|
||||
lastPromptTotal: null,
|
||||
lastGenerationTotal: null,
|
||||
lastAt: null,
|
||||
promptRate: 0,
|
||||
generationRate: 0,
|
||||
promptTotal: 0,
|
||||
generationTotal: 0,
|
||||
samples: [],
|
||||
error: null,
|
||||
},
|
||||
}
|
||||
return globalThis.__openshellControlLiveTelemetry
|
||||
}
|
||||
|
||||
function now() {
|
||||
return Date.now()
|
||||
}
|
||||
|
||||
function prune(cutoff = now() - WINDOW_MS) {
|
||||
const { events } = state()
|
||||
while (events.length > 0 && events[0].at < cutoff) events.shift()
|
||||
}
|
||||
|
||||
export function recordLiveTelemetryEvent(
|
||||
kind: TelemetryEventKind,
|
||||
options?: Omit<TelemetryEvent, "kind" | "at" | "count"> & { count?: number },
|
||||
) {
|
||||
const { events } = state()
|
||||
events.push({
|
||||
kind,
|
||||
at: now(),
|
||||
count: Math.max(0, options?.count ?? 1),
|
||||
sandboxId: options?.sandboxId ?? null,
|
||||
serverId: options?.serverId ?? null,
|
||||
toolName: options?.toolName ?? null,
|
||||
})
|
||||
prune()
|
||||
}
|
||||
|
||||
export function startLiveTelemetryTask(
|
||||
kind: TelemetryEventKind,
|
||||
options?: Omit<ActiveTask, "id" | "kind" | "startedAt">,
|
||||
) {
|
||||
const telemetryState = state()
|
||||
const id = telemetryState.nextTaskId
|
||||
telemetryState.nextTaskId += 1
|
||||
telemetryState.activeTasks.set(id, {
|
||||
id,
|
||||
kind,
|
||||
startedAt: now(),
|
||||
sandboxId: options?.sandboxId ?? null,
|
||||
serverId: options?.serverId ?? null,
|
||||
toolName: options?.toolName ?? null,
|
||||
})
|
||||
return () => {
|
||||
state().activeTasks.delete(id)
|
||||
}
|
||||
}
|
||||
|
||||
function rateFor(kind: TelemetryEventKind, windowMs: number) {
|
||||
const cutoff = now() - windowMs
|
||||
return state().events
|
||||
.filter((event) => event.kind === kind && event.at >= cutoff)
|
||||
.reduce((total, event) => total + event.count, 0) / (windowMs / 1000)
|
||||
}
|
||||
|
||||
function countFor(kind: TelemetryEventKind, windowMs: number) {
|
||||
const cutoff = now() - windowMs
|
||||
return state().events
|
||||
.filter((event) => event.kind === kind && event.at >= cutoff)
|
||||
.reduce((total, event) => total + event.count, 0)
|
||||
}
|
||||
|
||||
function tokenMetricsUrl() {
|
||||
const explicit = process.env.LIVE_TELEMETRY_TOKEN_METRICS_URL
|
||||
|| process.env.VLLM_METRICS_URL
|
||||
|| process.env.NEMOCLAW_VLLM_METRICS_URL
|
||||
if (explicit?.trim()) return explicit.trim()
|
||||
|
||||
const base = process.env.VLLM_BASE_URL
|
||||
|| process.env.OPENAI_BASE_URL
|
||||
|| process.env.MCP_PREFLIGHT_LLM_BASE_URL
|
||||
if (base?.trim()) {
|
||||
try {
|
||||
const url = new URL(base)
|
||||
url.pathname = "/metrics"
|
||||
url.search = ""
|
||||
url.hash = ""
|
||||
return url.toString()
|
||||
} catch {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
return "http://localhost:8000/metrics"
|
||||
}
|
||||
|
||||
function sumPrometheusMetrics(text: string, names: string[]) {
|
||||
const wanted = new Set(names)
|
||||
let total = 0
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.trim()
|
||||
if (!line || line.startsWith("#")) continue
|
||||
const match = line.match(/^([A-Za-z_:][A-Za-z0-9_:]*)(?:\{[^}]*\})?\s+(-?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)$/i)
|
||||
if (!match || !wanted.has(match[1])) continue
|
||||
const value = Number(match[2])
|
||||
if (Number.isFinite(value)) total += value
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
function sumSamples(samples: LiveTelemetryState["tokenMetrics"]["samples"], field: "promptDelta" | "generationDelta") {
|
||||
return samples.reduce((total, sample) => total + sample[field], 0)
|
||||
}
|
||||
|
||||
function sampleSeconds(samples: LiveTelemetryState["tokenMetrics"]["samples"]) {
|
||||
return samples.reduce((total, sample) => total + sample.elapsedSeconds, 0)
|
||||
}
|
||||
|
||||
async function refreshTokenMetrics() {
|
||||
const telemetryState = state()
|
||||
const endpoint = tokenMetricsUrl()
|
||||
if (!endpoint) {
|
||||
telemetryState.tokenMetrics = {
|
||||
...telemetryState.tokenMetrics,
|
||||
source: "not_configured",
|
||||
endpoint: "",
|
||||
promptRate: 0,
|
||||
generationRate: 0,
|
||||
error: "No token metrics endpoint configured",
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timeout = setTimeout(() => controller.abort(), 1500)
|
||||
try {
|
||||
const response = await fetch(endpoint, {
|
||||
cache: "no-store",
|
||||
signal: controller.signal,
|
||||
})
|
||||
if (!response.ok) throw new Error(`metrics returned HTTP ${response.status}`)
|
||||
const metrics = await response.text()
|
||||
const promptTotal = sumPrometheusMetrics(metrics, ["vllm:prompt_tokens_total"])
|
||||
const generationTotal = sumPrometheusMetrics(metrics, ["vllm:generation_tokens_total"])
|
||||
const at = now()
|
||||
const previousPromptTotal = telemetryState.tokenMetrics.lastPromptTotal
|
||||
const previousGenerationTotal = telemetryState.tokenMetrics.lastGenerationTotal
|
||||
const previousAt = telemetryState.tokenMetrics.lastAt
|
||||
const elapsedSeconds = previousAt ? (at - previousAt) / 1000 : 0
|
||||
const promptDelta = previousPromptTotal === null ? 0 : Math.max(0, promptTotal - previousPromptTotal)
|
||||
const generationDelta = previousGenerationTotal === null ? 0 : Math.max(0, generationTotal - previousGenerationTotal)
|
||||
const previousSamples = telemetryState.tokenMetrics.samples || []
|
||||
const samples = elapsedSeconds > 0
|
||||
? [...previousSamples, { at, promptDelta, generationDelta, elapsedSeconds }]
|
||||
: previousSamples
|
||||
const cutoff = at - WINDOW_MS
|
||||
const recentSamples = samples.filter((sample) => sample.at >= cutoff)
|
||||
const recentSeconds = sampleSeconds(recentSamples)
|
||||
|
||||
telemetryState.tokenMetrics = {
|
||||
source: "vllm",
|
||||
endpoint,
|
||||
lastPromptTotal: promptTotal,
|
||||
lastGenerationTotal: generationTotal,
|
||||
lastAt: at,
|
||||
promptRate: recentSeconds > 0 ? sumSamples(recentSamples, "promptDelta") / recentSeconds : 0,
|
||||
generationRate: recentSeconds > 0 ? sumSamples(recentSamples, "generationDelta") / recentSeconds : 0,
|
||||
promptTotal,
|
||||
generationTotal,
|
||||
samples: recentSamples,
|
||||
error: null,
|
||||
}
|
||||
} catch (error) {
|
||||
telemetryState.tokenMetrics = {
|
||||
...telemetryState.tokenMetrics,
|
||||
source: telemetryState.tokenMetrics.lastGenerationTotal === null ? "unavailable" : telemetryState.tokenMetrics.source,
|
||||
endpoint,
|
||||
promptRate: 0,
|
||||
generationRate: 0,
|
||||
error: error instanceof Error ? error.message : "Token metrics unavailable",
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
}
|
||||
|
||||
export async function readLiveTelemetrySnapshot() {
|
||||
await refreshTokenMetrics()
|
||||
const snapshotAt = now()
|
||||
prune(snapshotAt - WINDOW_MS)
|
||||
const windowMs = WINDOW_MS
|
||||
const active = Array.from(state().activeTasks.values())
|
||||
const tokenMetrics = state().tokenMetrics
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
timestamp: new Date(snapshotAt).toISOString(),
|
||||
windowMs,
|
||||
transactionsPerSecond: rateFor("transaction", windowMs),
|
||||
mcpRequestsPerSecond: rateFor("mcp_request", windowMs),
|
||||
mcpCallsPerSecond: rateFor("mcp_call", windowMs),
|
||||
tokensPerSecond: tokenMetrics.generationRate,
|
||||
promptTokensPerSecond: tokenMetrics.promptRate,
|
||||
generationTokensPerSecond: tokenMetrics.generationRate,
|
||||
tokenSource: tokenMetrics.source,
|
||||
tokenEndpoint: tokenMetrics.endpoint,
|
||||
tokenError: tokenMetrics.error,
|
||||
concurrentTasks: active.length,
|
||||
activeTasks: active.map((task) => ({
|
||||
kind: task.kind,
|
||||
ageMs: snapshotAt - task.startedAt,
|
||||
sandboxId: task.sandboxId,
|
||||
serverId: task.serverId,
|
||||
toolName: task.toolName,
|
||||
})),
|
||||
totals: {
|
||||
transactions: countFor("transaction", windowMs),
|
||||
mcpRequests: countFor("mcp_request", windowMs),
|
||||
mcpCalls: countFor("mcp_call", windowMs),
|
||||
mcpLists: countFor("mcp_list", windowMs),
|
||||
mcpErrors: countFor("mcp_error", windowMs),
|
||||
tokens: tokenMetrics.promptTotal + tokenMetrics.generationTotal,
|
||||
promptTokens: tokenMetrics.promptTotal,
|
||||
generationTokens: tokenMetrics.generationTotal,
|
||||
},
|
||||
}
|
||||
}
|
||||
+42
-20
@@ -2,6 +2,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"
|
||||
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"
|
||||
import { HOST_PATH } from "./hostCommands"
|
||||
import { recordLiveTelemetryEvent, startLiveTelemetryTask } from "./liveTelemetry"
|
||||
import type { McpServerInstall } from "./mcpServerStore"
|
||||
|
||||
const MCP_BROKER_REQUEST_TIMEOUT_MS = Number.parseInt(process.env.MCP_BROKER_REQUEST_TIMEOUT_MS || "45000", 10)
|
||||
@@ -55,16 +56,26 @@ async function withMcpClient<T>(server: McpServerInstall, fn: (client: Client) =
|
||||
}
|
||||
|
||||
export async function listBrokerServerTools(server: McpServerInstall) {
|
||||
return withMcpClient(server, async (client) => {
|
||||
const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS })
|
||||
return result.tools.map((tool) => ({
|
||||
name: tool.name,
|
||||
title: tool.title,
|
||||
description: tool.description,
|
||||
inputSchema: tool.inputSchema,
|
||||
annotations: tool.annotations,
|
||||
}))
|
||||
})
|
||||
const stopTask = startLiveTelemetryTask("mcp_list", { serverId: server.id })
|
||||
recordLiveTelemetryEvent("transaction", { serverId: server.id })
|
||||
try {
|
||||
return await withMcpClient(server, async (client) => {
|
||||
const result = await client.listTools({}, { timeout: MCP_BROKER_REQUEST_TIMEOUT_MS })
|
||||
recordLiveTelemetryEvent("mcp_list", { serverId: server.id })
|
||||
return result.tools.map((tool) => ({
|
||||
name: tool.name,
|
||||
title: tool.title,
|
||||
description: tool.description,
|
||||
inputSchema: tool.inputSchema,
|
||||
annotations: tool.annotations,
|
||||
}))
|
||||
})
|
||||
} catch (error) {
|
||||
recordLiveTelemetryEvent("mcp_error", { serverId: server.id })
|
||||
throw error
|
||||
} finally {
|
||||
stopTask()
|
||||
}
|
||||
}
|
||||
|
||||
export async function callBrokerServerTool(
|
||||
@@ -72,14 +83,25 @@ export async function callBrokerServerTool(
|
||||
toolName: string,
|
||||
toolArguments: Record<string, unknown>,
|
||||
) {
|
||||
return withMcpClient(server, async (client) => (
|
||||
client.callTool(
|
||||
{
|
||||
name: toolName,
|
||||
arguments: toolArguments,
|
||||
},
|
||||
undefined,
|
||||
{ timeout: MCP_BROKER_REQUEST_TIMEOUT_MS },
|
||||
)
|
||||
))
|
||||
const stopTask = startLiveTelemetryTask("mcp_call", { serverId: server.id, toolName })
|
||||
recordLiveTelemetryEvent("transaction", { serverId: server.id, toolName })
|
||||
try {
|
||||
const result = await withMcpClient(server, async (client) => (
|
||||
client.callTool(
|
||||
{
|
||||
name: toolName,
|
||||
arguments: toolArguments,
|
||||
},
|
||||
undefined,
|
||||
{ timeout: MCP_BROKER_REQUEST_TIMEOUT_MS },
|
||||
)
|
||||
))
|
||||
recordLiveTelemetryEvent("mcp_call", { serverId: server.id, toolName })
|
||||
return result
|
||||
} catch (error) {
|
||||
recordLiveTelemetryEvent("mcp_error", { serverId: server.id, toolName })
|
||||
throw error
|
||||
} finally {
|
||||
stopTask()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { callBrokerServerTool, listBrokerServerTools } from "./mcpBrokerClient"
|
||||
import { recordLiveTelemetryEvent } from "./liveTelemetry"
|
||||
import { listAllowedBrokerServers, type SandboxMcpBrokerSession } from "./mcpBrokerStore"
|
||||
import type { McpServerInstall } from "./mcpServerStore"
|
||||
|
||||
@@ -114,6 +115,8 @@ async function handleBrokerMcpJsonRpcMessage(session: SandboxMcpBrokerSession, m
|
||||
const id = hasJsonRpcId(request) ? request.id ?? null : null
|
||||
const method = typeof request.method === "string" ? request.method : ""
|
||||
const isNotification = !hasJsonRpcId(request)
|
||||
recordLiveTelemetryEvent("transaction", { sandboxId: session.sandboxId })
|
||||
recordLiveTelemetryEvent("mcp_request", { sandboxId: session.sandboxId })
|
||||
|
||||
try {
|
||||
switch (method) {
|
||||
|
||||
Reference in New Issue
Block a user