fleet(tower): 0.18 — Tasks-View (Spool/Dead-Letter/Alarm) + Task-Detail-Modal + /healthz + /api/status; Fleet-Views 0.13-0.18 erstmals versioniert
This commit is contained in:
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* fleet/fleetSource.ts — zentrale Datenquellen-Adapter für Fleet-Views (Design-Doc §2).
|
||||
*
|
||||
* Server-seitig (Next-API-Routen), KEIN Browser-Zugriff auf Cluster-Ressourcen.
|
||||
* Fail-loud: Quelle tot -> Exception -> 500 mit Klartext statt leeres UI (§5.5).
|
||||
*
|
||||
* Datenquellen:
|
||||
* - org.yaml + bifrost-vks.yaml: CM-Mount (read-only)
|
||||
* - Token je Agent: Prometheus /api/v1/query (bifrost_*_tokens_total, 2m-Rate)
|
||||
* - Heartbeat: Beacon /metrics
|
||||
* - Spool-Journal + Inbox: CephFS-RO-Mount (fleet-spool-PVC)
|
||||
* - Sandbox-Status: provisioning-MCP get_sandbox_status
|
||||
* - Board/Kanban + Owner-Submit: board-service-MCP
|
||||
*/
|
||||
import { readFileSync, existsSync, readdirSync } from "node:fs"
|
||||
import path from "node:path"
|
||||
import { parse as parseYaml } from "yaml"
|
||||
|
||||
export const FLEET_ORG_PATH = process.env.FLEET_ORG_YAML_PATH || "/etc/fleet/org.yaml"
|
||||
export const FLEET_VKS_PATH = process.env.FLEET_VKS_YAML_PATH || "/etc/fleet/bifrost-vks.yaml"
|
||||
export const FLEET_SPOOL_DIR = process.env.FLEET_SPOOL_DIR || "/srv/fleet/tasks"
|
||||
export const FLEET_PROM_URL = (process.env.FLEET_PROM_URL || "http://10.0.30.202:9090").replace(/\/$/, "")
|
||||
export const FLEET_BEACON_URL = (process.env.FLEET_BEACON_URL || "http://fleet-beacon.fleet.svc:8000").replace(/\/$/, "")
|
||||
export const FLEET_BOARD_MCP_URL =
|
||||
process.env.FLEET_BOARD_MCP_URL || "http://fleet-board-service.fleet.svc:8080/mcp"
|
||||
export const FLEET_PROVISIONING_MCP_URL =
|
||||
process.env.FLEET_PROVISIONING_MCP_URL || "http://fleet-provisioning.fleet.svc:8080/mcp"
|
||||
|
||||
// ---------------------------------------------------------------- org.yaml
|
||||
|
||||
export type OrgMember = {
|
||||
agent_id: string
|
||||
role: string
|
||||
al: number
|
||||
parent: string | null
|
||||
wave: number
|
||||
function: string
|
||||
title?: string
|
||||
}
|
||||
|
||||
export type OrgDoc = { members: OrgMember[] }
|
||||
|
||||
function readYamlDoc(p: string): Record<string, unknown> {
|
||||
if (!existsSync(p)) {
|
||||
throw new Error(`FLEET-Datenquelle fehlt: ${p} (CM gemountet?)`)
|
||||
}
|
||||
const doc = parseYaml(readFileSync(p, "utf8"))
|
||||
if (!doc || typeof doc !== "object") {
|
||||
throw new Error(`FLEET-Datenquelle unparsebar: ${p}`)
|
||||
}
|
||||
return doc as Record<string, unknown>
|
||||
}
|
||||
|
||||
/** org.yaml lesen (fail loud, kein Cache — Datei ist klein und statisch je Deploy). */
|
||||
export function loadOrg(): OrgDoc {
|
||||
const spec = readYamlDoc(FLEET_ORG_PATH).spec as { members?: OrgMember[] } | undefined
|
||||
const members = spec?.members
|
||||
if (!Array.isArray(members) || members.length === 0) {
|
||||
throw new Error(`org.yaml enthaelt keine spec.members (${FLEET_ORG_PATH})`)
|
||||
}
|
||||
for (const m of members) {
|
||||
if (!m.agent_id) throw new Error(`org.yaml Member ohne agent_id (${FLEET_ORG_PATH})`)
|
||||
}
|
||||
return { members }
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ bifrost-vks
|
||||
|
||||
export type VkMapping = { agent_id: string; vk_name: string; model: string; profile: string }
|
||||
|
||||
/** VK<->Agent-Mapping aus bifrost-vks.yaml. */
|
||||
export function loadVkMapping(): VkMapping[] {
|
||||
const spec = readYamlDoc(FLEET_VKS_PATH).spec as { members?: VkMapping[] } | undefined
|
||||
const members = spec?.members
|
||||
if (!Array.isArray(members) || members.length === 0) {
|
||||
throw new Error(`bifrost-vks.yaml enthaelt keine spec.members (${FLEET_VKS_PATH})`)
|
||||
}
|
||||
return members
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ utils
|
||||
|
||||
const globalCache = new Map<string, { value: unknown; expires: number }>()
|
||||
|
||||
/** Kleiner TTL-Cache (Server-seite; Views pollen 5 s — Prom/Beacon-TTL 2 s). */
|
||||
export function cached<T>(key: string, ttlMs: number, fn: () => T): T {
|
||||
const hit = globalCache.get(key) as { value: T; expires: number } | undefined
|
||||
if (hit && hit.expires > Date.now()) return hit.value
|
||||
const value = fn()
|
||||
globalCache.set(key, { value, expires: Date.now() + ttlMs })
|
||||
return value
|
||||
}
|
||||
|
||||
export async function fetchText(url: string, timeoutMs = 8000): Promise<string> {
|
||||
const ctrl = new AbortController()
|
||||
const timer = setTimeout(() => ctrl.abort(), timeoutMs)
|
||||
try {
|
||||
const res = await fetch(url, { signal: ctrl.signal, cache: "no-store" })
|
||||
if (!res.ok) {
|
||||
throw new Error(`HTTP ${res.status} von ${url}`)
|
||||
}
|
||||
return await res.text()
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------- Prometheus tok/s
|
||||
|
||||
export type TokenRates = Record<string, { in: number; out: number }>
|
||||
|
||||
/** tok/s je VK (2-Minuten-Rate) aus Prometheus.
|
||||
*
|
||||
* AGGREGATION IST PFLICHT: `rate()` liefert eine Serie je Label-Kombination
|
||||
* (method, model, provider, fallback_index, ...). Ein simples
|
||||
* `out[vk].in = v` ueberschreibt die echte Rate mit der letzten Serie — und das
|
||||
* ist fast immer eine 0. Folge: Org-Chart zeigt 0 tok/s, obwohl Bifrost
|
||||
* Millionen Tokens zaehlt (Befund 29.10.2026: devops-v2 8,5 Mio in 30 min,
|
||||
* Tower 0). Deshalb `sum by (virtual_key_name)` in der Query UND additive
|
||||
* Auswertung im Code (doppelte Absicherung).
|
||||
*/
|
||||
export async function loadTokenRates(): Promise<TokenRates> {
|
||||
const query = (metric: string) =>
|
||||
`${FLEET_PROM_URL}/api/v1/query?query=${encodeURIComponent(`sum by (virtual_key_name) (rate(${metric}[2m]))`)}`
|
||||
const [inBody, outBody] = await Promise.all([
|
||||
fetchText(query("bifrost_input_tokens_total")),
|
||||
fetchText(query("bifrost_output_tokens_total")),
|
||||
])
|
||||
const pick = (body: string): Array<{ vk: string; v: number }> => {
|
||||
const parsed = JSON.parse(body)
|
||||
if (parsed.status !== "success") {
|
||||
throw new Error(`Prometheus-Query nicht success: ${parsed.status || body.slice(0, 120)}`)
|
||||
}
|
||||
return (parsed.data?.result || []).map((r: { metric: Record<string, string>; value: [number, string] }) => ({
|
||||
vk: String(r.metric.virtual_key_name || ""),
|
||||
v: Number(r.value[1]) || 0,
|
||||
}))
|
||||
}
|
||||
const out: TokenRates = {}
|
||||
for (const { vk, v } of pick(inBody)) {
|
||||
if (vk) (out[vk] = out[vk] || { in: 0, out: 0 }).in += v
|
||||
}
|
||||
for (const { vk, v } of pick(outBody)) {
|
||||
if (vk) (out[vk] = out[vk] || { in: 0, out: 0 }).out += v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ Beacon /metrics
|
||||
|
||||
export type HeartbeatAges = Record<string, number>
|
||||
|
||||
/** Heartbeat-Alter je Agent aus Beacon /metrics. */
|
||||
export async function loadHeartbeatAges(): Promise<HeartbeatAges> {
|
||||
const body = await fetchText(`${FLEET_BEACON_URL}/metrics`)
|
||||
const out: HeartbeatAges = {}
|
||||
for (const line of body.split("\n")) {
|
||||
const m = line.match(/^beacon_last_heartbeat_age_seconds\{agent="([^"]+)"\}\s+([0-9.e+-]+)$/)
|
||||
if (m) out[m[1]] = Number(m[2])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Spool-Journal
|
||||
|
||||
export type JournalLine = {
|
||||
time_epoch?: number
|
||||
event?: string
|
||||
task_id?: string
|
||||
from?: string
|
||||
to?: string
|
||||
type?: string
|
||||
priority?: string
|
||||
status?: string
|
||||
file?: string
|
||||
[k: string]: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* Spool-Journal-Tail (letzte N JSON-Lines aus _journal/journal.logl).
|
||||
* Fail loud wenn Journal fehlt — es MUSS existieren (Spool-PVC ro gemountet).
|
||||
*/
|
||||
export function readJournalTail(limit = 200): JournalLine[] {
|
||||
const jpath = path.join(FLEET_SPOOL_DIR, "_journal", "journal.logl")
|
||||
if (!existsSync(jpath)) {
|
||||
throw new Error(`Spool-Journal fehlt: ${jpath} (fleet-spool-PVC ro gemountet?)`)
|
||||
}
|
||||
const raw = readFileSync(jpath, "utf8").trim()
|
||||
if (!raw) return []
|
||||
const lines = raw.split("\n")
|
||||
const tail = lines.slice(-limit)
|
||||
const out: JournalLine[] = []
|
||||
for (const line of tail) {
|
||||
try {
|
||||
out.push(JSON.parse(line) as JournalLine)
|
||||
} catch {
|
||||
// Journal ist append-only; eine kaputte Zeile wirft NICHT das ganze UI um,
|
||||
// wird aber sichtbar markiert statt verschluckt (fail-loud-lesend).
|
||||
out.push({ event: "journal_parse_error", raw: line.slice(0, 200) })
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Inbox-Groesse je to-Agent: anzahl *unbearbeiteter* Task-Dateien im Spool-Root. */
|
||||
export function countInbox(): Record<string, number> {
|
||||
const out: Record<string, number> = {}
|
||||
if (!existsSync(FLEET_SPOOL_DIR)) {
|
||||
throw new Error(`Spool-Dir fehlt: ${FLEET_SPOOL_DIR} (fleet-spool-PVC ro gemountet?)`)
|
||||
}
|
||||
for (const f of readdirSync(FLEET_SPOOL_DIR)) {
|
||||
if (!f.endsWith(".yaml")) continue
|
||||
try {
|
||||
const doc = parseYaml(readFileSync(path.join(FLEET_SPOOL_DIR, f), "utf8")) as { to?: string; status?: string }
|
||||
if (!doc?.to) continue
|
||||
if (doc.status && doc.status !== "submitted" && doc.status !== "running") continue
|
||||
out[doc.to] = (out[doc.to] || 0) + 1
|
||||
} catch {
|
||||
// unparsebare Task-Datei: zaehlt unter _invalid-artefakt, ignoriert hier nicht
|
||||
// grob — wird im Journal als rejected sichtbar sein.
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Chat-Threads: Task-Dateien mit type=chat gruppiert nach thread_id. */
|
||||
export type ChatThread = {
|
||||
thread_id: string
|
||||
topic: string
|
||||
messages: Array<{ task_id: string; from: string; to: string; body: string; created: number }>
|
||||
}
|
||||
|
||||
export function readChatThreads(): ChatThread[] {
|
||||
if (!existsSync(FLEET_SPOOL_DIR)) {
|
||||
throw new Error(`Spool-Dir fehlt: ${FLEET_SPOOL_DIR}`)
|
||||
}
|
||||
const threads = new Map<string, ChatThread>()
|
||||
for (const f of readdirSync(FLEET_SPOOL_DIR)) {
|
||||
if (!f.endsWith(".yaml")) continue
|
||||
try {
|
||||
const doc = parseYaml(readFileSync(path.join(FLEET_SPOOL_DIR, f), "utf8")) as {
|
||||
task_id?: string
|
||||
from?: string
|
||||
to?: string
|
||||
type?: string
|
||||
thread_id?: string
|
||||
topic?: string
|
||||
body?: string
|
||||
created?: number
|
||||
}
|
||||
if (doc?.type !== "chat" || !doc.thread_id) continue
|
||||
const t = threads.get(doc.thread_id) || { thread_id: doc.thread_id, topic: doc.topic || "", messages: [] }
|
||||
t.messages.push({
|
||||
task_id: String(doc.task_id || f),
|
||||
from: String(doc.from || ""),
|
||||
to: String(doc.to || ""),
|
||||
body: String(doc.body || ""),
|
||||
created: Number(doc.created || 0),
|
||||
})
|
||||
threads.set(doc.thread_id, t)
|
||||
} catch {
|
||||
// kaputte Datei: skip (im Journal sichtbar)
|
||||
}
|
||||
}
|
||||
const list = [...threads.values()]
|
||||
for (const t of list) t.messages.sort((a, b) => a.created - b.created)
|
||||
return list
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ kube-state-metrics
|
||||
|
||||
export type SandboxFact = {
|
||||
agent_id: string
|
||||
phase: string
|
||||
ready: boolean
|
||||
restarts: number
|
||||
pod_age_s: number
|
||||
node: string
|
||||
cpu_request_milli: number
|
||||
mem_request_mb: number
|
||||
image_digest: string | null
|
||||
}
|
||||
|
||||
function parsePromQuery(body: string): Array<{ metric: Record<string, string>; value: [number, string] }> {
|
||||
const parsed = JSON.parse(body)
|
||||
if (parsed.status !== "success") {
|
||||
throw new Error(`Prometheus-Query nicht success: ${parsed.status || String(body).slice(0, 120)}`)
|
||||
}
|
||||
return (parsed.data?.result || []) as Array<{ metric: Record<string, string>; value: [number, string] }>
|
||||
}
|
||||
|
||||
async function promQueryRaw(promql: string): Promise<Array<{ metric: Record<string, string>; value: [number, string] }>> {
|
||||
const body = await fetchText(`${FLEET_PROM_URL}/api/v1/query?query=${encodeURIComponent(promql)}`)
|
||||
return parsePromQuery(body)
|
||||
}
|
||||
|
||||
/**
|
||||
* Sandbox-Pod-Facts je Agent aus kube-state-metrics (read-only PromQL).
|
||||
* Pods heissen default--<agent_id> in NS openshell (K8s-Driver-Konvention).
|
||||
* Fail-loud: Prometheus tot -> Exception (kein leeres UI).
|
||||
*/
|
||||
export async function loadSandboxFacts(): Promise<{
|
||||
sandboxes: SandboxFact[]
|
||||
fleet_image_digest: string | null
|
||||
}> {
|
||||
const agentIds = loadOrg().members.map((m) => m.agent_id)
|
||||
const podSel = agentIds.map((a) => `default--${a}`).join("|")
|
||||
const ns = "openshell"
|
||||
const [phaseRes, readyRes, restartRes, startRes, infoRes, reqRes, imgRes] = await Promise.all([
|
||||
promQueryRaw(`kube_pod_status_phase{namespace="${ns}",pod=~"${podSel}"}`),
|
||||
promQueryRaw(`kube_pod_container_status_ready{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
|
||||
promQueryRaw(`kube_pod_container_status_restarts_total{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
|
||||
promQueryRaw(`kube_pod_start_time{namespace="${ns}",pod=~"${podSel}"}`),
|
||||
promQueryRaw(`kube_pod_info{namespace="${ns}",pod=~"${podSel}"}`),
|
||||
promQueryRaw(`kube_pod_container_resource_requests{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
|
||||
promQueryRaw(`kube_pod_container_info{namespace="${ns}",pod=~"${podSel}",container="agent"}`),
|
||||
])
|
||||
const agentOf = (pod: string | undefined): string | null => {
|
||||
if (!pod) return null
|
||||
const m = pod.match(/^default--(.+)$/)
|
||||
return m ? m[1] : null
|
||||
}
|
||||
const now = Date.now() / 1000
|
||||
// kube_pod_status_phase liefert EINE Zeile je Phase je Pod -> je Pod einmal
|
||||
const pods = [...new Set(phaseRes.map((r) => r.metric.pod).filter(Boolean))] as string[]
|
||||
const out: SandboxFact[] = []
|
||||
for (const pod of pods) {
|
||||
const agent = agentOf(pod)
|
||||
if (!agent) continue
|
||||
const phase =
|
||||
phaseRes.find((x) => x.metric.pod === pod && Number(x.value[1]) === 1)?.metric.phase ||
|
||||
phaseRes.find((x) => x.metric.pod === pod)?.metric.phase ||
|
||||
"Unknown"
|
||||
const readyRow = readyRes.find((x) => x.metric.pod === pod && Number(x.value[1]) === 1)
|
||||
const restartRow = restartRes.find((x) => x.metric.pod === pod)
|
||||
const startRow = startRes.find((x) => x.metric.pod === pod)
|
||||
const infoRow = infoRes.find((x) => x.metric.pod === pod)
|
||||
const cpuRow = reqRes.find((x) => x.metric.pod === pod && x.metric.resource === "cpu")
|
||||
const memRow = reqRes.find((x) => x.metric.pod === pod && x.metric.resource === "memory")
|
||||
const imgRow = imgRes.find((x) => x.metric.pod === pod)
|
||||
out.push({
|
||||
agent_id: agent,
|
||||
phase,
|
||||
ready: Boolean(readyRow),
|
||||
restarts: restartRow ? Math.round(Number(restartRow.value[1])) : 0,
|
||||
pod_age_s: startRow ? Math.max(0, Math.round(now - Number(startRow.value[1]))) : 0,
|
||||
node: infoRow?.metric.node || "?",
|
||||
cpu_request_milli: cpuRow ? Math.round(Number(cpuRow.value[1]) * 1000) : 0,
|
||||
mem_request_mb: memRow ? Math.round(Number(memRow.value[1]) / (1024 * 1024)) : 0,
|
||||
image_digest: imgRow?.metric.image_id?.match(/sha256:[a-f0-9]+/)?.[0] || null,
|
||||
})
|
||||
}
|
||||
const fleetRow = imgRes.find((x) => x.metric.pod?.startsWith("default--"))
|
||||
return { sandboxes: out, fleet_image_digest: fleetRow?.metric.image_id?.match(/sha256:[a-f0-9]+/)?.[0] || null }
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- MCP-Client
|
||||
|
||||
export type McpCallResult = { ok: boolean; data?: unknown; error?: string }
|
||||
|
||||
type McpResponsePayload = {
|
||||
result?: { content?: Array<{ type: string; text?: string }> }
|
||||
error?: { message?: string }
|
||||
}
|
||||
|
||||
/** Streamable-HTTP-MCP-Call gegen eine Fleet-Facility (mcp 2.3.0, stateless). */
|
||||
export async function mcpCallTool(serverUrl: string, tool: string, args: Record<string, unknown> = {}, timeoutMs = 20000): Promise<McpCallResult> {
|
||||
const ctrl = new AbortController()
|
||||
const timer = setTimeout(() => ctrl.abort(), timeoutMs)
|
||||
try {
|
||||
const headers = {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/event-stream",
|
||||
}
|
||||
// 1) initialize (stateless_http) — noetig bevor tools/call
|
||||
const initRes = await fetch(serverUrl, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: 1,
|
||||
method: "initialize",
|
||||
params: {
|
||||
protocolVersion: "2025-06-18",
|
||||
capabilities: {},
|
||||
clientInfo: { name: "fleet-tower", version: "0.13" },
|
||||
},
|
||||
}),
|
||||
signal: ctrl.signal,
|
||||
cache: "no-store",
|
||||
})
|
||||
if (!initRes.ok) throw new Error(`MCP initialize HTTP ${initRes.status} von ${serverUrl}`)
|
||||
// 2) tools/call
|
||||
const callRes = await fetch(serverUrl, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: 2, method: "tools/call", params: { name: tool, arguments: args } }),
|
||||
signal: ctrl.signal,
|
||||
cache: "no-store",
|
||||
})
|
||||
if (!callRes.ok) throw new Error(`MCP tools/call HTTP ${callRes.status} von ${serverUrl}`)
|
||||
const text = await callRes.text()
|
||||
// Antwort kann JSON oder SSE sein — letzte data:-Zeile ist das Result.
|
||||
let payload: unknown
|
||||
const ct = callRes.headers.get("content-type") || ""
|
||||
if (ct.includes("text/event-stream")) {
|
||||
const dataLines = text.split("\n").filter((l) => l.startsWith("data:"))
|
||||
if (dataLines.length === 0) throw new Error(`MCP-Antwort ohne data:-Zeile von ${serverUrl}`)
|
||||
payload = JSON.parse(dataLines[dataLines.length - 1].slice(5).trim())
|
||||
} else {
|
||||
payload = JSON.parse(text)
|
||||
}
|
||||
const obj = payload as McpResponsePayload
|
||||
if (obj?.error) return { ok: false, error: String(obj.error.message || JSON.stringify(obj.error)) }
|
||||
const textContent = obj?.result?.content?.find((c) => c.type === "text")?.text
|
||||
return { ok: true, data: textContent ? JSON.parse(textContent) : obj?.result }
|
||||
} catch (err) {
|
||||
return { ok: false, error: err instanceof Error ? err.message : String(err) }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Betriebsreife-Endpunkte für den Tower (Karte it1-tower-03-healthz-status).
|
||||
*
|
||||
* /healthz — Liveness/Readiness für Uptime-Monitoring (HTTP 200 = Kern ok,
|
||||
* 503 = eine Kernquelle tot). JSON, nie HTML.
|
||||
* /api/status — maschinenlesbares JSON: Version, Uptime, Zustand der
|
||||
* Datenquellen (Spool-PVC, org.yaml, Prometheus, Beacon) mit
|
||||
* Latenz und Klartext-Fehler.
|
||||
*
|
||||
* Beide Endpunkte sind fail-loud-lesend: eine tote Quelle macht den Endpunkt
|
||||
* NICHT 500, sondern sichtbar degraded (Uptime-Monitoring braucht eine Antwort).
|
||||
*/
|
||||
export const TOWER_VERSION = process.env.FLEET_TOWER_VERSION || "0.18"
|
||||
export const TOWER_STARTED_AT_MS = Date.now()
|
||||
|
||||
export type SourceState = {
|
||||
ok: boolean
|
||||
latency_ms: number
|
||||
error: string
|
||||
}
|
||||
|
||||
export function uptimeSeconds(): number {
|
||||
return Math.round((Date.now() - TOWER_STARTED_AT_MS) / 1000)
|
||||
}
|
||||
|
||||
/** Quelle mit Timeout prüfen; wirft nie. */
|
||||
export async function probe(url: string, timeoutMs = 3000): Promise<SourceState> {
|
||||
const t0 = Date.now()
|
||||
const ctrl = new AbortController()
|
||||
const timer = setTimeout(() => ctrl.abort(), timeoutMs)
|
||||
try {
|
||||
const res = await fetch(url, { signal: ctrl.signal, cache: "no-store" })
|
||||
return {
|
||||
ok: res.ok,
|
||||
latency_ms: Date.now() - t0,
|
||||
error: res.ok ? "" : `HTTP ${res.status}`,
|
||||
}
|
||||
} catch (e) {
|
||||
return {
|
||||
ok: false,
|
||||
latency_ms: Date.now() - t0,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
/**
|
||||
* fleet/taskSource.ts — Task-Envelope-Leser für die Task-Detail-Ansicht.
|
||||
*
|
||||
* Karte tower-task-detail (Change-Order 2026-10-30 aus devops-Sandbox,
|
||||
* /sandbox/fleet-change-orders/tower-task-detail/): Klick auf einen Eintrag der
|
||||
* Listen Spool-Reifegrad / Alarm-Feed / Dead-Letter zeigt den VOLLEN
|
||||
* Task-Envelope (from/to/type/status/priority/created/Alter/topic/body/...).
|
||||
*
|
||||
* Datenquelle: Spool-PVC ro (FLEET_SPOOL_DIR), dieselbe Quelle wie
|
||||
* countInbox()/readJournalTail(). Kein Egress, keine Secrets, kein neuer Grant.
|
||||
*
|
||||
* Abnahme-Regeln aus der Change-Order:
|
||||
* - Fehlende Felder -> "-" (missing_fields listet sie zusaetzlich).
|
||||
* - found:false (synthetisches Ereignis, z.B. Beacon-Event mit ref=agent_id)
|
||||
* liefert alle Kernfelder als "-" und die angefragte id, KEIN Crash.
|
||||
* - Body UNVERAENDERT (keine Maskierung) — XSS-Escaping nur im Panel.
|
||||
* - Nie 5xx fuer unbekannte ids; tote Quelle -> degraded statt Throw.
|
||||
*/
|
||||
import { readFileSync, existsSync, readdirSync, statSync } from "node:fs"
|
||||
import path from "node:path"
|
||||
import { parse as parseYaml } from "yaml"
|
||||
import { FLEET_SPOOL_DIR, readJournalTail, type JournalLine } from "./fleetSource"
|
||||
|
||||
export const MISSING = "-"
|
||||
const DEAD_LETTER_DAYS = 7
|
||||
const DETAIL_STALE_SECONDS = Number(process.env.FLEET_DETAIL_STALE_SECONDS || "900")
|
||||
|
||||
const CORE_FIELDS = [
|
||||
"task_id", "from", "to", "type", "status", "priority", "created",
|
||||
"created_iso", "age_seconds", "age_human", "topic", "body", "thread_id",
|
||||
"karte_id", "reply_to", "vs", "claimed_at", "status_label",
|
||||
] as const
|
||||
|
||||
const STATUS_LABEL: Record<string, string> = {
|
||||
submitted: "eingereicht", working: "in Arbeit", open: "offen",
|
||||
completed: "abgeschlossen", failed: "fehlgeschlagen", blocked: "blockiert",
|
||||
}
|
||||
|
||||
export type TaskEnvelope = {
|
||||
task_id: string
|
||||
from: string
|
||||
to: string
|
||||
type: string
|
||||
status: string
|
||||
priority: string
|
||||
created: number | null
|
||||
created_iso: string
|
||||
age_seconds: number | null
|
||||
age_human: string
|
||||
topic: string
|
||||
body: string
|
||||
thread_id: string
|
||||
karte_id: string
|
||||
reply_to: string
|
||||
vs: string
|
||||
claimed_at: string
|
||||
status_label: string
|
||||
file: string
|
||||
}
|
||||
|
||||
export type TaskDetail = {
|
||||
generated_at: string
|
||||
now: number
|
||||
source: string
|
||||
refresh_seconds: number
|
||||
entry_kind: string
|
||||
found: boolean
|
||||
query_task_id: string
|
||||
task: Record<string, string | number | null>
|
||||
missing_fields: string[]
|
||||
extra_fields: Record<string, unknown>
|
||||
raw_keys: string[]
|
||||
body_note: string
|
||||
degraded: boolean
|
||||
degraded_reason: string
|
||||
data_ts: string | number | null
|
||||
data_age_seconds: number | null
|
||||
stale: boolean
|
||||
error: string
|
||||
}
|
||||
|
||||
function iso(epoch: number | null): string {
|
||||
return epoch === null ? MISSING : new Date(epoch * 1000).toISOString()
|
||||
}
|
||||
|
||||
function humanize(seconds: number | null): string {
|
||||
if (seconds === null) return MISSING
|
||||
const s = Math.max(0, Math.floor(seconds))
|
||||
const d = Math.floor(s / 86400)
|
||||
const h = Math.floor((s % 86400) / 3600)
|
||||
const m = Math.floor((s % 3600) / 60)
|
||||
if (d) return `${d}d ${h}h`
|
||||
if (h) return `${h}h ${m}m`
|
||||
return `${m}m`
|
||||
}
|
||||
|
||||
function toEpoch(value: unknown): number | null {
|
||||
if (value === null || value === undefined || typeof value === "boolean") return null
|
||||
if (typeof value === "number") return Number.isFinite(value) ? Math.floor(value) : null
|
||||
const s = String(value).trim()
|
||||
if (!s) return null
|
||||
if (/^\d+$/.test(s)) return Number(s)
|
||||
const t = Date.parse(s)
|
||||
return Number.isNaN(t) ? null : Math.floor(t / 1000)
|
||||
}
|
||||
|
||||
function str(value: unknown): string {
|
||||
if (value === null || value === undefined) return MISSING
|
||||
if (typeof value === "string") return value.trim() === "" ? MISSING : value
|
||||
if (typeof value === "object") return JSON.stringify(value)
|
||||
return String(value)
|
||||
}
|
||||
|
||||
function normalizeEnvelope(doc: Record<string, unknown>, now: number, file: string): TaskEnvelope {
|
||||
const taskId = String(doc.task_id ?? doc.id ?? "")
|
||||
const created = toEpoch(doc.created ?? doc.created_at)
|
||||
const age = created === null ? null : Math.max(0, now - created)
|
||||
const status = String(doc.status ?? "").toLowerCase()
|
||||
const claimedRaw = doc.claimed_at ?? doc.working_since ?? doc.claimed
|
||||
return {
|
||||
task_id: taskId || MISSING,
|
||||
from: str(doc.from ?? doc.from_agent),
|
||||
to: str(doc.to),
|
||||
type: str(doc.type),
|
||||
status: status || MISSING,
|
||||
priority: str(doc.priority),
|
||||
created,
|
||||
created_iso: iso(created),
|
||||
age_seconds: age,
|
||||
age_human: humanize(age),
|
||||
topic: str(doc.topic),
|
||||
body: str(doc.body),
|
||||
thread_id: str(doc.thread_id),
|
||||
karte_id: str(doc.karte_id),
|
||||
reply_to: str(doc.reply_to),
|
||||
vs: str(doc.vs),
|
||||
claimed_at: claimedRaw === undefined || claimedRaw === null ? MISSING : iso(toEpoch(claimedRaw)),
|
||||
status_label: STATUS_LABEL[status] ?? (status || MISSING),
|
||||
file,
|
||||
}
|
||||
}
|
||||
|
||||
type RawEnvelope = { doc: Record<string, unknown>; envelope: TaskEnvelope }
|
||||
|
||||
function readAllEnvelopes(now: number): RawEnvelope[] {
|
||||
if (!existsSync(FLEET_SPOOL_DIR)) {
|
||||
throw new Error(`Spool-Dir fehlt: ${FLEET_SPOOL_DIR} (fleet-spool-PVC ro gemountet?)`)
|
||||
}
|
||||
const out: RawEnvelope[] = []
|
||||
for (const f of readdirSync(FLEET_SPOOL_DIR)) {
|
||||
if (!f.endsWith(".yaml")) continue
|
||||
const full = path.join(FLEET_SPOOL_DIR, f)
|
||||
try {
|
||||
if (!statSync(full).isFile()) continue
|
||||
const doc = parseYaml(readFileSync(full, "utf8")) as Record<string, unknown> | null
|
||||
if (!doc || typeof doc !== "object") continue
|
||||
if (!doc.task_id && !doc.id) continue
|
||||
out.push({ doc, envelope: normalizeEnvelope(doc, now, f) })
|
||||
} catch {
|
||||
// Unparsebare Datei: nicht Teil der Ansicht — erscheint im Journal als rejected.
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Spool-Reifegrad: offene Envelopes (submitted/working), aelteste zuerst. */
|
||||
export function listSpoolTasks(): TaskEnvelope[] {
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
const open = readAllEnvelopes(now).filter(({ envelope }) =>
|
||||
["submitted", "working", "running", "open"].includes(envelope.status))
|
||||
return open.map((r) => r.envelope).sort((a, b) => (b.age_seconds ?? 0) - (a.age_seconds ?? 0))
|
||||
}
|
||||
|
||||
/** Dead-Letter: fehlgeschlagene Envelopes der letzten 7 Tage, aelteste zuerst. */
|
||||
export function listDeadLetterTasks(): TaskEnvelope[] {
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
const cutoff = now - DEAD_LETTER_DAYS * 86400
|
||||
return readAllEnvelopes(now)
|
||||
.filter(({ envelope }) => envelope.status === "failed")
|
||||
.filter(({ envelope }) => envelope.created !== null && envelope.created >= cutoff)
|
||||
.map((r) => r.envelope)
|
||||
.sort((a, b) => (b.age_seconds ?? 0) - (a.age_seconds ?? 0))
|
||||
}
|
||||
|
||||
export type AlarmRow = {
|
||||
time_epoch: number | null
|
||||
time_iso: string
|
||||
age_human: string
|
||||
event: string
|
||||
ref: string
|
||||
from: string
|
||||
to: string
|
||||
task_id: string
|
||||
note: string
|
||||
}
|
||||
|
||||
const ALARM_EVENT_RE = /deadman|alarm|stall|beacon|gateway|reject|invalid|orphan|escalat|blocked|timeout|fail|_warn/i
|
||||
|
||||
/** Alarm-Feed: Stoerungs-/Zustandsereignisse aus dem Spool-Journal (letzte 24 h). */
|
||||
export function listAlarmEvents(limit = 120): AlarmRow[] {
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
const cutoff = now - 24 * 3600
|
||||
const lines: JournalLine[] = readJournalTail(1500)
|
||||
const rows: AlarmRow[] = []
|
||||
for (const line of lines) {
|
||||
const event = String(line.event ?? "")
|
||||
if (!event || !ALARM_EVENT_RE.test(event)) continue
|
||||
const t = toEpoch(line.time_epoch)
|
||||
if (t !== null && t < cutoff) continue
|
||||
rows.push({
|
||||
time_epoch: t,
|
||||
time_iso: iso(t),
|
||||
age_human: humanize(t === null ? null : now - t),
|
||||
event,
|
||||
ref: String(line.task_id ?? line.ref ?? line.agent_id ?? line.agent ?? MISSING),
|
||||
from: String(line.from ?? line.by ?? MISSING),
|
||||
to: String(line.to ?? MISSING),
|
||||
task_id: String(line.task_id ?? MISSING),
|
||||
note: String(line.note ?? line.reason ?? ""),
|
||||
})
|
||||
}
|
||||
return rows.slice(-limit).reverse()
|
||||
}
|
||||
|
||||
/** Vollstaendiger Envelope fuer die Detailansicht. Wirft NICHT bei unbekannter id. */
|
||||
export function getTaskDetail(queryId: string, kind = ""): TaskDetail {
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
const base: TaskDetail = {
|
||||
generated_at: new Date(now * 1000).toISOString(),
|
||||
now,
|
||||
source: FLEET_SPOOL_DIR,
|
||||
refresh_seconds: 30,
|
||||
entry_kind: kind || "unknown",
|
||||
found: false,
|
||||
query_task_id: queryId,
|
||||
task: Object.fromEntries(CORE_FIELDS.map((k) => [k, MISSING])),
|
||||
missing_fields: [...CORE_FIELDS],
|
||||
extra_fields: {},
|
||||
raw_keys: [],
|
||||
body_note: "Body unveraendert aus dem Spool (keine Maskierung).",
|
||||
degraded: false,
|
||||
degraded_reason: "",
|
||||
data_ts: null,
|
||||
data_age_seconds: null,
|
||||
stale: false,
|
||||
error: "",
|
||||
}
|
||||
if (!queryId) {
|
||||
base.error = "Parameter id fehlt"
|
||||
return base
|
||||
}
|
||||
if (!existsSync(FLEET_SPOOL_DIR)) {
|
||||
base.degraded = true
|
||||
base.degraded_reason = "quelle-tot"
|
||||
base.error = `Spool-Dir fehlt: ${FLEET_SPOOL_DIR}`
|
||||
return base
|
||||
}
|
||||
try {
|
||||
const wanted = queryId.trim().toLowerCase()
|
||||
const hits = readAllEnvelopes(now).filter(({ envelope }) =>
|
||||
envelope.task_id.toLowerCase() === wanted)
|
||||
if (hits.length === 0) {
|
||||
base.task.task_id = queryId
|
||||
return base
|
||||
}
|
||||
const best = hits.sort((a, b) => (b.envelope.created ?? 0) - (a.envelope.created ?? 0))[0]
|
||||
const env = best.envelope
|
||||
const task: Record<string, string | number | null> = {}
|
||||
const missing: string[] = []
|
||||
for (const k of CORE_FIELDS) {
|
||||
const v = env[k as keyof TaskEnvelope]
|
||||
if (v === MISSING || v === null || v === "") {
|
||||
task[k] = MISSING
|
||||
missing.push(k)
|
||||
} else {
|
||||
task[k] = v as string | number
|
||||
}
|
||||
}
|
||||
const extra: Record<string, unknown> = {}
|
||||
const known = new Set<string>([...CORE_FIELDS, "id", "file", "created_at", "working_since", "claimed"])
|
||||
for (const [k, v] of Object.entries(best.doc)) {
|
||||
if (!known.has(k)) extra[k] = v
|
||||
}
|
||||
base.found = true
|
||||
base.task = task
|
||||
base.missing_fields = missing
|
||||
base.extra_fields = extra
|
||||
base.raw_keys = Object.keys(best.doc).sort()
|
||||
base.data_ts = env.created
|
||||
base.data_age_seconds = env.age_seconds
|
||||
base.stale = env.age_seconds !== null && env.age_seconds > DETAIL_STALE_SECONDS
|
||||
return base
|
||||
} catch (e) {
|
||||
base.degraded = true
|
||||
base.degraded_reason = "lesefehler"
|
||||
base.error = e instanceof Error ? e.message : String(e)
|
||||
return base
|
||||
}
|
||||
}
|
||||
|
||||
export type TaskOverview = {
|
||||
generated_at: string
|
||||
spool: TaskEnvelope[]
|
||||
dead_letter: TaskEnvelope[]
|
||||
alarm: AlarmRow[]
|
||||
counts: { spool: number; dead_letter: number; alarm: number }
|
||||
degraded: boolean
|
||||
degraded_reason: string
|
||||
}
|
||||
|
||||
/** Alle drei Listen in einem Call (Panel-Initialisierung + 30s-Refresh). */
|
||||
export function loadTaskOverview(): TaskOverview {
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
const out: TaskOverview = {
|
||||
generated_at: new Date(now * 1000).toISOString(),
|
||||
spool: [], dead_letter: [], alarm: [],
|
||||
counts: { spool: 0, dead_letter: 0, alarm: 0 },
|
||||
degraded: false, degraded_reason: "",
|
||||
}
|
||||
try {
|
||||
out.spool = listSpoolTasks()
|
||||
out.dead_letter = listDeadLetterTasks()
|
||||
out.alarm = listAlarmEvents()
|
||||
out.counts = { spool: out.spool.length, dead_letter: out.dead_letter.length, alarm: out.alarm.length }
|
||||
} catch (e) {
|
||||
out.degraded = true
|
||||
out.degraded_reason = e instanceof Error ? e.message : String(e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user