/** * fleet/fleetSource.ts — zentrale Datenquellen-Adapter für Fleet-Views (Design-Doc §2). * * Server-seitig (Next-API-Routen), KEIN Browser-Zugriff auf Cluster-Ressourcen. * Fail-loud: Quelle tot -> Exception -> 500 mit Klartext statt leeres UI (§5.5). * * Datenquellen: * - org.yaml + bifrost-vks.yaml: CM-Mount (read-only) * - Token je Agent: Prometheus /api/v1/query (bifrost_*_tokens_total, 2m-Rate) * - Heartbeat: Beacon /metrics * - Spool-Journal + Inbox: CephFS-RO-Mount (fleet-spool-PVC) * - Sandbox-Status: provisioning-MCP get_sandbox_status * - Board/Kanban + Owner-Submit: board-service-MCP */ import { readFileSync, existsSync, readdirSync } from "node:fs" import path from "node:path" import { parse as parseYaml } from "yaml" export const FLEET_ORG_PATH = process.env.FLEET_ORG_YAML_PATH || "/etc/fleet/org.yaml" export const FLEET_VKS_PATH = process.env.FLEET_VKS_YAML_PATH || "/etc/fleet/bifrost-vks.yaml" export const FLEET_SPOOL_DIR = process.env.FLEET_SPOOL_DIR || "/srv/fleet/tasks" export const FLEET_PROM_URL = (process.env.FLEET_PROM_URL || "http://10.0.30.202:9090").replace(/\/$/, "") export const FLEET_BEACON_URL = (process.env.FLEET_BEACON_URL || "http://fleet-beacon.fleet.svc:8000").replace(/\/$/, "") export const FLEET_BOARD_MCP_URL = process.env.FLEET_BOARD_MCP_URL || "http://fleet-board-service.fleet.svc:8080/mcp" export const FLEET_PROVISIONING_MCP_URL = process.env.FLEET_PROVISIONING_MCP_URL || "http://fleet-provisioning.fleet.svc:8080/mcp" // ---------------------------------------------------------------- org.yaml export type OrgMember = { agent_id: string role: string al: number parent: string | null wave: number function: string title?: string } export type OrgDoc = { members: OrgMember[] } function readYamlDoc(p: string): Record { if (!existsSync(p)) { throw new Error(`FLEET-Datenquelle fehlt: ${p} (CM gemountet?)`) } const doc = parseYaml(readFileSync(p, "utf8")) if (!doc || typeof doc !== "object") { throw new Error(`FLEET-Datenquelle unparsebar: ${p}`) } return doc as Record } /** org.yaml lesen (fail loud, kein Cache — Datei ist klein und statisch je Deploy). */ export function loadOrg(): OrgDoc { const spec = readYamlDoc(FLEET_ORG_PATH).spec as { members?: OrgMember[] } | undefined const members = spec?.members if (!Array.isArray(members) || members.length === 0) { throw new Error(`org.yaml enthaelt keine spec.members (${FLEET_ORG_PATH})`) } for (const m of members) { if (!m.agent_id) throw new Error(`org.yaml Member ohne agent_id (${FLEET_ORG_PATH})`) } return { members } } // ------------------------------------------------------------ bifrost-vks export type VkMapping = { agent_id: string; vk_name: string; model: string; profile: string } /** VK<->Agent-Mapping aus bifrost-vks.yaml. */ export function loadVkMapping(): VkMapping[] { const spec = readYamlDoc(FLEET_VKS_PATH).spec as { members?: VkMapping[] } | undefined const members = spec?.members if (!Array.isArray(members) || members.length === 0) { throw new Error(`bifrost-vks.yaml enthaelt keine spec.members (${FLEET_VKS_PATH})`) } return members } // ------------------------------------------------------------------ utils const globalCache = new Map() /** Kleiner TTL-Cache (Server-seite; Views pollen 5 s — Prom/Beacon-TTL 2 s). */ export function cached(key: string, ttlMs: number, fn: () => T): T { const hit = globalCache.get(key) as { value: T; expires: number } | undefined if (hit && hit.expires > Date.now()) return hit.value const value = fn() globalCache.set(key, { value, expires: Date.now() + ttlMs }) return value } export async function fetchText(url: string, timeoutMs = 8000): Promise { const ctrl = new AbortController() const timer = setTimeout(() => ctrl.abort(), timeoutMs) try { const res = await fetch(url, { signal: ctrl.signal, cache: "no-store" }) if (!res.ok) { throw new Error(`HTTP ${res.status} von ${url}`) } return await res.text() } finally { clearTimeout(timer) } } // ------------------------------------------------------- Prometheus tok/s export type TokenRates = Record /** tok/s je VK (2-Minuten-Rate) aus Prometheus. * * AGGREGATION IST PFLICHT: `rate()` liefert eine Serie je Label-Kombination * (method, model, provider, fallback_index, ...). Ein simples * `out[vk].in = v` ueberschreibt die echte Rate mit der letzten Serie — und das * ist fast immer eine 0. Folge: Org-Chart zeigt 0 tok/s, obwohl Bifrost * Millionen Tokens zaehlt (Befund 29.10.2026: devops-v2 8,5 Mio in 30 min, * Tower 0). Deshalb `sum by (virtual_key_name)` in der Query UND additive * Auswertung im Code (doppelte Absicherung). */ export async function loadTokenRates(): Promise { const query = (metric: string) => `${FLEET_PROM_URL}/api/v1/query?query=${encodeURIComponent(`sum by (virtual_key_name) (rate(${metric}[2m]))`)}` const [inBody, outBody] = await Promise.all([ fetchText(query("bifrost_input_tokens_total")), fetchText(query("bifrost_output_tokens_total")), ]) const pick = (body: string): Array<{ vk: string; v: number }> => { const parsed = JSON.parse(body) if (parsed.status !== "success") { throw new Error(`Prometheus-Query nicht success: ${parsed.status || body.slice(0, 120)}`) } return (parsed.data?.result || []).map((r: { metric: Record; value: [number, string] }) => ({ vk: String(r.metric.virtual_key_name || ""), v: Number(r.value[1]) || 0, })) } const out: TokenRates = {} for (const { vk, v } of pick(inBody)) { if (vk) (out[vk] = out[vk] || { in: 0, out: 0 }).in += v } for (const { vk, v } of pick(outBody)) { if (vk) (out[vk] = out[vk] || { in: 0, out: 0 }).out += v } return out } // ------------------------------------------------------------ Beacon /metrics export type HeartbeatAges = Record /** Heartbeat-Alter je Agent aus Beacon /metrics. */ export async function loadHeartbeatAges(): Promise { const body = await fetchText(`${FLEET_BEACON_URL}/metrics`) const out: HeartbeatAges = {} for (const line of body.split("\n")) { const m = line.match(/^beacon_last_heartbeat_age_seconds\{agent="([^"]+)"\}\s+([0-9.e+-]+)$/) if (m) out[m[1]] = Number(m[2]) } return out } // ---------------------------------------------------------------- Spool-Journal export type JournalLine = { time_epoch?: number event?: string task_id?: string from?: string to?: string type?: string priority?: string status?: string file?: string [k: string]: unknown } /** * Spool-Journal-Tail (letzte N JSON-Lines aus _journal/journal.logl). * Fail loud wenn Journal fehlt — es MUSS existieren (Spool-PVC ro gemountet). */ export function readJournalTail(limit = 200): JournalLine[] { const jpath = path.join(FLEET_SPOOL_DIR, "_journal", "journal.logl") if (!existsSync(jpath)) { throw new Error(`Spool-Journal fehlt: ${jpath} (fleet-spool-PVC ro gemountet?)`) } const raw = readFileSync(jpath, "utf8").trim() if (!raw) return [] const lines = raw.split("\n") const tail = lines.slice(-limit) const out: JournalLine[] = [] for (const line of tail) { try { out.push(JSON.parse(line) as JournalLine) } catch { // Journal ist append-only; eine kaputte Zeile wirft NICHT das ganze UI um, // wird aber sichtbar markiert statt verschluckt (fail-loud-lesend). out.push({ event: "journal_parse_error", raw: line.slice(0, 200) }) } } return out } /** Inbox-Groesse je to-Agent: anzahl *unbearbeiteter* Task-Dateien im Spool-Root. */ export function countInbox(): Record { const out: Record = {} if (!existsSync(FLEET_SPOOL_DIR)) { throw new Error(`Spool-Dir fehlt: ${FLEET_SPOOL_DIR} (fleet-spool-PVC ro gemountet?)`) } for (const f of readdirSync(FLEET_SPOOL_DIR)) { if (!f.endsWith(".yaml")) continue try { const doc = parseYaml(readFileSync(path.join(FLEET_SPOOL_DIR, f), "utf8")) as { to?: string; status?: string } if (!doc?.to) continue if (doc.status && doc.status !== "submitted" && doc.status !== "running") continue out[doc.to] = (out[doc.to] || 0) + 1 } catch { // unparsebare Task-Datei: zaehlt unter _invalid-artefakt, ignoriert hier nicht // grob — wird im Journal als rejected sichtbar sein. } } return out } /** Chat-Threads: Task-Dateien mit type=chat gruppiert nach thread_id. */ export type ChatThread = { thread_id: string topic: string messages: Array<{ task_id: string; from: string; to: string; body: string; created: number }> } export function readChatThreads(): ChatThread[] { if (!existsSync(FLEET_SPOOL_DIR)) { throw new Error(`Spool-Dir fehlt: ${FLEET_SPOOL_DIR}`) } const threads = new Map() for (const f of readdirSync(FLEET_SPOOL_DIR)) { if (!f.endsWith(".yaml")) continue try { const doc = parseYaml(readFileSync(path.join(FLEET_SPOOL_DIR, f), "utf8")) as { task_id?: string from?: string to?: string type?: string thread_id?: string topic?: string body?: string created?: number } if (doc?.type !== "chat" || !doc.thread_id) continue const t = threads.get(doc.thread_id) || { thread_id: doc.thread_id, topic: doc.topic || "", messages: [] } t.messages.push({ task_id: String(doc.task_id || f), from: String(doc.from || ""), to: String(doc.to || ""), body: String(doc.body || ""), created: Number(doc.created || 0), }) threads.set(doc.thread_id, t) } catch { // kaputte Datei: skip (im Journal sichtbar) } } const list = [...threads.values()] for (const t of list) t.messages.sort((a, b) => a.created - b.created) return list } // ------------------------------------------------------------ kube-state-metrics export type SandboxFact = { agent_id: string phase: string ready: boolean restarts: number pod_age_s: number node: string cpu_request_milli: number mem_request_mb: number image_digest: string | null } function parsePromQuery(body: string): Array<{ metric: Record; value: [number, string] }> { const parsed = JSON.parse(body) if (parsed.status !== "success") { throw new Error(`Prometheus-Query nicht success: ${parsed.status || String(body).slice(0, 120)}`) } return (parsed.data?.result || []) as Array<{ metric: Record; value: [number, string] }> } async function promQueryRaw(promql: string): Promise; value: [number, string] }>> { const body = await fetchText(`${FLEET_PROM_URL}/api/v1/query?query=${encodeURIComponent(promql)}`) return parsePromQuery(body) } /** * Sandbox-Pod-Facts je Agent aus kube-state-metrics (read-only PromQL). * Pods heissen default-- in NS openshell (K8s-Driver-Konvention). * Fail-loud: Prometheus tot -> Exception (kein leeres UI). */ export async function loadSandboxFacts(): Promise<{ sandboxes: SandboxFact[] fleet_image_digest: string | null }> { const agentIds = loadOrg().members.map((m) => m.agent_id) const podSel = agentIds.map((a) => `default--${a}`).join("|") const ns = "openshell" const [phaseRes, readyRes, restartRes, startRes, infoRes, reqRes, imgRes] = await Promise.all([ promQueryRaw(`kube_pod_status_phase{namespace="${ns}",pod=~"${podSel}"}`), promQueryRaw(`kube_pod_container_status_ready{namespace="${ns}",pod=~"${podSel}",container="agent"}`), promQueryRaw(`kube_pod_container_status_restarts_total{namespace="${ns}",pod=~"${podSel}",container="agent"}`), promQueryRaw(`kube_pod_start_time{namespace="${ns}",pod=~"${podSel}"}`), promQueryRaw(`kube_pod_info{namespace="${ns}",pod=~"${podSel}"}`), promQueryRaw(`kube_pod_container_resource_requests{namespace="${ns}",pod=~"${podSel}",container="agent"}`), promQueryRaw(`kube_pod_container_info{namespace="${ns}",pod=~"${podSel}",container="agent"}`), ]) const agentOf = (pod: string | undefined): string | null => { if (!pod) return null const m = pod.match(/^default--(.+)$/) return m ? m[1] : null } const now = Date.now() / 1000 // kube_pod_status_phase liefert EINE Zeile je Phase je Pod -> je Pod einmal const pods = [...new Set(phaseRes.map((r) => r.metric.pod).filter(Boolean))] as string[] const out: SandboxFact[] = [] for (const pod of pods) { const agent = agentOf(pod) if (!agent) continue const phase = phaseRes.find((x) => x.metric.pod === pod && Number(x.value[1]) === 1)?.metric.phase || phaseRes.find((x) => x.metric.pod === pod)?.metric.phase || "Unknown" const readyRow = readyRes.find((x) => x.metric.pod === pod && Number(x.value[1]) === 1) const restartRow = restartRes.find((x) => x.metric.pod === pod) const startRow = startRes.find((x) => x.metric.pod === pod) const infoRow = infoRes.find((x) => x.metric.pod === pod) const cpuRow = reqRes.find((x) => x.metric.pod === pod && x.metric.resource === "cpu") const memRow = reqRes.find((x) => x.metric.pod === pod && x.metric.resource === "memory") const imgRow = imgRes.find((x) => x.metric.pod === pod) out.push({ agent_id: agent, phase, ready: Boolean(readyRow), restarts: restartRow ? Math.round(Number(restartRow.value[1])) : 0, pod_age_s: startRow ? Math.max(0, Math.round(now - Number(startRow.value[1]))) : 0, node: infoRow?.metric.node || "?", cpu_request_milli: cpuRow ? Math.round(Number(cpuRow.value[1]) * 1000) : 0, mem_request_mb: memRow ? Math.round(Number(memRow.value[1]) / (1024 * 1024)) : 0, image_digest: imgRow?.metric.image_id?.match(/sha256:[a-f0-9]+/)?.[0] || null, }) } const fleetRow = imgRes.find((x) => x.metric.pod?.startsWith("default--")) return { sandboxes: out, fleet_image_digest: fleetRow?.metric.image_id?.match(/sha256:[a-f0-9]+/)?.[0] || null } } // ------------------------------------------------------------- MCP-Client export type McpCallResult = { ok: boolean; data?: unknown; error?: string } type McpResponsePayload = { result?: { content?: Array<{ type: string; text?: string }> } error?: { message?: string } } /** Streamable-HTTP-MCP-Call gegen eine Fleet-Facility (mcp 2.3.0, stateless). */ export async function mcpCallTool(serverUrl: string, tool: string, args: Record = {}, timeoutMs = 20000): Promise { const ctrl = new AbortController() const timer = setTimeout(() => ctrl.abort(), timeoutMs) try { const headers = { "Content-Type": "application/json", Accept: "application/json, text/event-stream", } // 1) initialize (stateless_http) — noetig bevor tools/call const initRes = await fetch(serverUrl, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "fleet-tower", version: "0.13" }, }, }), signal: ctrl.signal, cache: "no-store", }) if (!initRes.ok) throw new Error(`MCP initialize HTTP ${initRes.status} von ${serverUrl}`) // 2) tools/call const callRes = await fetch(serverUrl, { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", id: 2, method: "tools/call", params: { name: tool, arguments: args } }), signal: ctrl.signal, cache: "no-store", }) if (!callRes.ok) throw new Error(`MCP tools/call HTTP ${callRes.status} von ${serverUrl}`) const text = await callRes.text() // Antwort kann JSON oder SSE sein — letzte data:-Zeile ist das Result. let payload: unknown const ct = callRes.headers.get("content-type") || "" if (ct.includes("text/event-stream")) { const dataLines = text.split("\n").filter((l) => l.startsWith("data:")) if (dataLines.length === 0) throw new Error(`MCP-Antwort ohne data:-Zeile von ${serverUrl}`) payload = JSON.parse(dataLines[dataLines.length - 1].slice(5).trim()) } else { payload = JSON.parse(text) } const obj = payload as McpResponsePayload if (obj?.error) return { ok: false, error: String(obj.error.message || JSON.stringify(obj.error)) } const textContent = obj?.result?.content?.find((c) => c.type === "text")?.text return { ok: true, data: textContent ? JSON.parse(textContent) : obj?.result } } catch (err) { return { ok: false, error: err instanceof Error ? err.message : String(err) } } finally { clearTimeout(timer) } }