Files

423 lines
17 KiB
TypeScript

/**
* fleet/fleetSource.ts — zentrale Datenquellen-Adapter für Fleet-Views (Design-Doc §2).
*
* Server-seitig (Next-API-Routen), KEIN Browser-Zugriff auf Cluster-Ressourcen.
* Fail-loud: Quelle tot -> Exception -> 500 mit Klartext statt leeres UI (§5.5).
*
* Datenquellen:
* - org.yaml + bifrost-vks.yaml: CM-Mount (read-only)
* - Token je Agent: Prometheus /api/v1/query (bifrost_*_tokens_total, 2m-Rate)
* - Heartbeat: Beacon /metrics
* - Spool-Journal + Inbox: CephFS-RO-Mount (fleet-spool-PVC)
* - Sandbox-Status: provisioning-MCP get_sandbox_status
* - Board/Kanban + Owner-Submit: board-service-MCP
*/
import { readFileSync, existsSync, readdirSync } from "node:fs"
import path from "node:path"
import { parse as parseYaml } from "yaml"
export const FLEET_ORG_PATH = process.env.FLEET_ORG_YAML_PATH || "/etc/fleet/org.yaml"
export const FLEET_VKS_PATH = process.env.FLEET_VKS_YAML_PATH || "/etc/fleet/bifrost-vks.yaml"
export const FLEET_SPOOL_DIR = process.env.FLEET_SPOOL_DIR || "/srv/fleet/tasks"
export const FLEET_PROM_URL = (process.env.FLEET_PROM_URL || "http://10.0.30.202:9090").replace(/\/$/, "")
export const FLEET_BEACON_URL = (process.env.FLEET_BEACON_URL || "http://fleet-beacon.fleet.svc:8000").replace(/\/$/, "")
export const FLEET_BOARD_MCP_URL =
process.env.FLEET_BOARD_MCP_URL || "http://fleet-board-service.fleet.svc:8080/mcp"
export const FLEET_PROVISIONING_MCP_URL =
process.env.FLEET_PROVISIONING_MCP_URL || "http://fleet-provisioning.fleet.svc:8080/mcp"
// ---------------------------------------------------------------- org.yaml
export type OrgMember = {
agent_id: string
role: string
al: number
parent: string | null
wave: number
function: string
title?: string
}
export type OrgDoc = { members: OrgMember[] }
function readYamlDoc(p: string): Record<string, unknown> {
if (!existsSync(p)) {
throw new Error(`FLEET-Datenquelle fehlt: ${p} (CM gemountet?)`)
}
const doc = parseYaml(readFileSync(p, "utf8"))
if (!doc || typeof doc !== "object") {
throw new Error(`FLEET-Datenquelle unparsebar: ${p}`)
}
return doc as Record<string, unknown>
}
/** org.yaml lesen (fail loud, kein Cache — Datei ist klein und statisch je Deploy). */
export function loadOrg(): OrgDoc {
const spec = readYamlDoc(FLEET_ORG_PATH).spec as { members?: OrgMember[] } | undefined
const members = spec?.members
if (!Array.isArray(members) || members.length === 0) {
throw new Error(`org.yaml enthaelt keine spec.members (${FLEET_ORG_PATH})`)
}
for (const m of members) {
if (!m.agent_id) throw new Error(`org.yaml Member ohne agent_id (${FLEET_ORG_PATH})`)
}
return { members }
}
// ------------------------------------------------------------ bifrost-vks
export type VkMapping = { agent_id: string; vk_name: string; model: string; profile: string }
/** VK<->Agent-Mapping aus bifrost-vks.yaml. */
export function loadVkMapping(): VkMapping[] {
const spec = readYamlDoc(FLEET_VKS_PATH).spec as { members?: VkMapping[] } | undefined
const members = spec?.members
if (!Array.isArray(members) || members.length === 0) {
throw new Error(`bifrost-vks.yaml enthaelt keine spec.members (${FLEET_VKS_PATH})`)
}
return members
}
// ------------------------------------------------------------------ utils
const globalCache = new Map<string, { value: unknown; expires: number }>()
/** Kleiner TTL-Cache (Server-seite; Views pollen 5 s — Prom/Beacon-TTL 2 s). */
export function cached<T>(key: string, ttlMs: number, fn: () => T): T {
const hit = globalCache.get(key) as { value: T; expires: number } | undefined
if (hit && hit.expires > Date.now()) return hit.value
const value = fn()
globalCache.set(key, { value, expires: Date.now() + ttlMs })
return value
}
export async function fetchText(url: string, timeoutMs = 8000): Promise<string> {
const ctrl = new AbortController()
const timer = setTimeout(() => ctrl.abort(), timeoutMs)
try {
const res = await fetch(url, { signal: ctrl.signal, cache: "no-store" })
if (!res.ok) {
throw new Error(`HTTP ${res.status} von ${url}`)
}
return await res.text()
} finally {
clearTimeout(timer)
}
}
// ------------------------------------------------------- Prometheus tok/s
export type TokenRates = Record<string, { in: number; out: number }>
/** tok/s je VK (2-Minuten-Rate) aus Prometheus.
*
* AGGREGATION IST PFLICHT: `rate()` liefert eine Serie je Label-Kombination
* (method, model, provider, fallback_index, ...). Ein simples
* `out[vk].in = v` ueberschreibt die echte Rate mit der letzten Serie — und das
* ist fast immer eine 0. Folge: Org-Chart zeigt 0 tok/s, obwohl Bifrost
* Millionen Tokens zaehlt (Befund 29.10.2026: devops-v2 8,5 Mio in 30 min,
* Tower 0). Deshalb `sum by (virtual_key_name)` in der Query UND additive
* Auswertung im Code (doppelte Absicherung).
*/
export async function loadTokenRates(): Promise<TokenRates> {
const query = (metric: string) =>
`${FLEET_PROM_URL}/api/v1/query?query=${encodeURIComponent(`sum by (virtual_key_name) (rate(${metric}[2m]))`)}`
const [inBody, outBody] = await Promise.all([
fetchText(query("bifrost_input_tokens_total")),
fetchText(query("bifrost_output_tokens_total")),
])
const pick = (body: string): Array<{ vk: string; v: number }> => {
const parsed = JSON.parse(body)
if (parsed.status !== "success") {
throw new Error(`Prometheus-Query nicht success: ${parsed.status || body.slice(0, 120)}`)
}
return (parsed.data?.result || []).map((r: { metric: Record<string, string>; value: [number, string] }) => ({
vk: String(r.metric.virtual_key_name || ""),
v: Number(r.value[1]) || 0,
}))
}
const out: TokenRates = {}
for (const { vk, v } of pick(inBody)) {
if (vk) (out[vk] = out[vk] || { in: 0, out: 0 }).in += v
}
for (const { vk, v } of pick(outBody)) {
if (vk) (out[vk] = out[vk] || { in: 0, out: 0 }).out += v
}
return out
}
// ------------------------------------------------------------ Beacon /metrics
export type HeartbeatAges = Record<string, number>
/** Heartbeat-Alter je Agent aus Beacon /metrics. */
export async function loadHeartbeatAges(): Promise<HeartbeatAges> {
const body = await fetchText(`${FLEET_BEACON_URL}/metrics`)
const out: HeartbeatAges = {}
for (const line of body.split("\n")) {
const m = line.match(/^beacon_last_heartbeat_age_seconds\{agent="([^"]+)"\}\s+([0-9.e+-]+)$/)
if (m) out[m[1]] = Number(m[2])
}
return out
}
// ---------------------------------------------------------------- Spool-Journal
export type JournalLine = {
time_epoch?: number
event?: string
task_id?: string
from?: string
to?: string
type?: string
priority?: string
status?: string
file?: string
[k: string]: unknown
}
/**
* Spool-Journal-Tail (letzte N JSON-Lines aus _journal/journal.logl).
* Fail loud wenn Journal fehlt — es MUSS existieren (Spool-PVC ro gemountet).
*/
export function readJournalTail(limit = 200): JournalLine[] {
const jpath = path.join(FLEET_SPOOL_DIR, "_journal", "journal.logl")
if (!existsSync(jpath)) {
throw new Error(`Spool-Journal fehlt: ${jpath} (fleet-spool-PVC ro gemountet?)`)
}
const raw = readFileSync(jpath, "utf8").trim()
if (!raw) return []
const lines = raw.split("\n")
const tail = lines.slice(-limit)
const out: JournalLine[] = []
for (const line of tail) {
try {
out.push(JSON.parse(line) as JournalLine)
} catch {
// Journal ist append-only; eine kaputte Zeile wirft NICHT das ganze UI um,
// wird aber sichtbar markiert statt verschluckt (fail-loud-lesend).
out.push({ event: "journal_parse_error", raw: line.slice(0, 200) })
}
}
return out
}
/** Inbox-Groesse je to-Agent: anzahl *unbearbeiteter* Task-Dateien im Spool-Root. */
export function countInbox(): Record<string, number> {
const out: Record<string, number> = {}
if (!existsSync(FLEET_SPOOL_DIR)) {
throw new Error(`Spool-Dir fehlt: ${FLEET_SPOOL_DIR} (fleet-spool-PVC ro gemountet?)`)
}
for (const f of readdirSync(FLEET_SPOOL_DIR)) {
if (!f.endsWith(".yaml")) continue
try {
const doc = parseYaml(readFileSync(path.join(FLEET_SPOOL_DIR, f), "utf8")) as { to?: string; status?: string }
if (!doc?.to) continue
if (doc.status && doc.status !== "submitted" && doc.status !== "running") continue
out[doc.to] = (out[doc.to] || 0) + 1
} catch {
// unparsebare Task-Datei: zaehlt unter _invalid-artefakt, ignoriert hier nicht
// grob — wird im Journal als rejected sichtbar sein.
}
}
return out
}
/** Chat-Threads: Task-Dateien mit type=chat gruppiert nach thread_id. */
export type ChatThread = {
thread_id: string
topic: string
messages: Array<{ task_id: string; from: string; to: string; body: string; created: number }>
}
export function readChatThreads(): ChatThread[] {
if (!existsSync(FLEET_SPOOL_DIR)) {
throw new Error(`Spool-Dir fehlt: ${FLEET_SPOOL_DIR}`)
}
const threads = new Map<string, ChatThread>()
for (const f of readdirSync(FLEET_SPOOL_DIR)) {
if (!f.endsWith(".yaml")) continue
try {
const doc = parseYaml(readFileSync(path.join(FLEET_SPOOL_DIR, f), "utf8")) as {
task_id?: string
from?: string
to?: string
type?: string
thread_id?: string
topic?: string
body?: string
created?: number
}
if (doc?.type !== "chat" || !doc.thread_id) continue
const t = threads.get(doc.thread_id) || { thread_id: doc.thread_id, topic: doc.topic || "", messages: [] }
t.messages.push({
task_id: String(doc.task_id || f),
from: String(doc.from || ""),
to: String(doc.to || ""),
body: String(doc.body || ""),
created: Number(doc.created || 0),
})
threads.set(doc.thread_id, t)
} catch {
// kaputte Datei: skip (im Journal sichtbar)
}
}
const list = [...threads.values()]
for (const t of list) t.messages.sort((a, b) => a.created - b.created)
return list
}
// ------------------------------------------------------------ kube-state-metrics
export type SandboxFact = {
agent_id: string
phase: string
ready: boolean
restarts: number
pod_age_s: number
node: string
cpu_request_milli: number
mem_request_mb: number
image_digest: string | null
}
function parsePromQuery(body: string): Array<{ metric: Record<string, string>; value: [number, string] }> {
const parsed = JSON.parse(body)
if (parsed.status !== "success") {
throw new Error(`Prometheus-Query nicht success: ${parsed.status || String(body).slice(0, 120)}`)
}
return (parsed.data?.result || []) as Array<{ metric: Record<string, string>; value: [number, string] }>
}
async function promQueryRaw(promql: string): Promise<Array<{ metric: Record<string, string>; value: [number, string] }>> {
const body = await fetchText(`${FLEET_PROM_URL}/api/v1/query?query=${encodeURIComponent(promql)}`)
return parsePromQuery(body)
}
/**
* Sandbox-Pod-Facts je Agent aus kube-state-metrics (read-only PromQL).
* Pods heissen default--<agent_id> in NS openshell (K8s-Driver-Konvention).
* Fail-loud: Prometheus tot -> Exception (kein leeres UI).
*/
export async function loadSandboxFacts(): Promise<{
sandboxes: SandboxFact[]
fleet_image_digest: string | null
}> {
const agentIds = loadOrg().members.map((m) => m.agent_id)
const podSel = agentIds.map((a) => `default--${a}`).join("|")
const ns = "openshell"
const [phaseRes, readyRes, restartRes, startRes, infoRes, reqRes, imgRes] = await Promise.all([
promQueryRaw(`kube_pod_status_phase{namespace="${ns}",pod=~"${podSel}"}`),
promQueryRaw(`kube_pod_container_status_ready{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
promQueryRaw(`kube_pod_container_status_restarts_total{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
promQueryRaw(`kube_pod_start_time{namespace="${ns}",pod=~"${podSel}"}`),
promQueryRaw(`kube_pod_info{namespace="${ns}",pod=~"${podSel}"}`),
promQueryRaw(`kube_pod_container_resource_requests{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
promQueryRaw(`kube_pod_container_info{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
])
const agentOf = (pod: string | undefined): string | null => {
if (!pod) return null
const m = pod.match(/^default--(.+)$/)
return m ? m[1] : null
}
const now = Date.now() / 1000
// kube_pod_status_phase liefert EINE Zeile je Phase je Pod -> je Pod einmal
const pods = [...new Set(phaseRes.map((r) => r.metric.pod).filter(Boolean))] as string[]
const out: SandboxFact[] = []
for (const pod of pods) {
const agent = agentOf(pod)
if (!agent) continue
const phase =
phaseRes.find((x) => x.metric.pod === pod && Number(x.value[1]) === 1)?.metric.phase ||
phaseRes.find((x) => x.metric.pod === pod)?.metric.phase ||
"Unknown"
const readyRow = readyRes.find((x) => x.metric.pod === pod && Number(x.value[1]) === 1)
const restartRow = restartRes.find((x) => x.metric.pod === pod)
const startRow = startRes.find((x) => x.metric.pod === pod)
const infoRow = infoRes.find((x) => x.metric.pod === pod)
const cpuRow = reqRes.find((x) => x.metric.pod === pod && x.metric.resource === "cpu")
const memRow = reqRes.find((x) => x.metric.pod === pod && x.metric.resource === "memory")
const imgRow = imgRes.find((x) => x.metric.pod === pod)
out.push({
agent_id: agent,
phase,
ready: Boolean(readyRow),
restarts: restartRow ? Math.round(Number(restartRow.value[1])) : 0,
pod_age_s: startRow ? Math.max(0, Math.round(now - Number(startRow.value[1]))) : 0,
node: infoRow?.metric.node || "?",
cpu_request_milli: cpuRow ? Math.round(Number(cpuRow.value[1]) * 1000) : 0,
mem_request_mb: memRow ? Math.round(Number(memRow.value[1]) / (1024 * 1024)) : 0,
image_digest: imgRow?.metric.image_id?.match(/sha256:[a-f0-9]+/)?.[0] || null,
})
}
const fleetRow = imgRes.find((x) => x.metric.pod?.startsWith("default--"))
return { sandboxes: out, fleet_image_digest: fleetRow?.metric.image_id?.match(/sha256:[a-f0-9]+/)?.[0] || null }
}
// ------------------------------------------------------------- MCP-Client
export type McpCallResult = { ok: boolean; data?: unknown; error?: string }
type McpResponsePayload = {
result?: { content?: Array<{ type: string; text?: string }> }
error?: { message?: string }
}
/** Streamable-HTTP-MCP-Call gegen eine Fleet-Facility (mcp 2.3.0, stateless). */
export async function mcpCallTool(serverUrl: string, tool: string, args: Record<string, unknown> = {}, timeoutMs = 20000): Promise<McpCallResult> {
const ctrl = new AbortController()
const timer = setTimeout(() => ctrl.abort(), timeoutMs)
try {
const headers = {
"Content-Type": "application/json",
Accept: "application/json, text/event-stream",
}
// 1) initialize (stateless_http) — noetig bevor tools/call
const initRes = await fetch(serverUrl, {
method: "POST",
headers,
body: JSON.stringify({
jsonrpc: "2.0",
id: 1,
method: "initialize",
params: {
protocolVersion: "2025-06-18",
capabilities: {},
clientInfo: { name: "fleet-tower", version: "0.13" },
},
}),
signal: ctrl.signal,
cache: "no-store",
})
if (!initRes.ok) throw new Error(`MCP initialize HTTP ${initRes.status} von ${serverUrl}`)
// 2) tools/call
const callRes = await fetch(serverUrl, {
method: "POST",
headers,
body: JSON.stringify({ jsonrpc: "2.0", id: 2, method: "tools/call", params: { name: tool, arguments: args } }),
signal: ctrl.signal,
cache: "no-store",
})
if (!callRes.ok) throw new Error(`MCP tools/call HTTP ${callRes.status} von ${serverUrl}`)
const text = await callRes.text()
// Antwort kann JSON oder SSE sein — letzte data:-Zeile ist das Result.
let payload: unknown
const ct = callRes.headers.get("content-type") || ""
if (ct.includes("text/event-stream")) {
const dataLines = text.split("\n").filter((l) => l.startsWith("data:"))
if (dataLines.length === 0) throw new Error(`MCP-Antwort ohne data:-Zeile von ${serverUrl}`)
payload = JSON.parse(dataLines[dataLines.length - 1].slice(5).trim())
} else {
payload = JSON.parse(text)
}
const obj = payload as McpResponsePayload
if (obj?.error) return { ok: false, error: String(obj.error.message || JSON.stringify(obj.error)) }
const textContent = obj?.result?.content?.find((c) => c.type === "text")?.text
return { ok: true, data: textContent ? JSON.parse(textContent) : obj?.result }
} catch (err) {
return { ok: false, error: err instanceof Error ? err.message : String(err) }
} finally {
clearTimeout(timer)
}
}