Initial commit: Hermes Agent Skills collection
This commit is contained in:
Debian
@@ -0,0 +1,213 @@
|
||||
# Prometheus Alerting Rules for Proxmox VE + Ceph + Galera + Blackbox
|
||||
#
|
||||
# Copy to /opt/monitoring/prometheus/rules/alerting_rules.yml (or equivalent
|
||||
# rules dir mounted into the prometheus container) and reload:
|
||||
# docker exec prometheus wget -qO- --post-data= http://localhost:9090/-/reload
|
||||
#
|
||||
# CRITICAL PITFALLS (see references/alertmanager-webhook-setup.md):
|
||||
# - pve_up includes ALL guests (including intentionally stopped ones).
|
||||
# Filter with id=~"node/.*" for node-down alerts only.
|
||||
# - pve_ha_state has 15 states × hundreds of guests = 600+ series.
|
||||
# Only alert on state=~"error|fence|freeze|gone".
|
||||
# - After changing rules, restart alertmanager to clear stale cached alerts.
|
||||
|
||||
groups:
|
||||
# === NODE DOWN ===
|
||||
- name: node_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: NodeDown
|
||||
expr: up{job="node_exporter"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} is down"
|
||||
description: "{{ $labels.instance }} has been unreachable for 2 minutes."
|
||||
|
||||
- alert: NodeHighCPU
|
||||
expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{job="node_exporter",mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU on {{ $labels.instance }}"
|
||||
description: "CPU usage above 85% for 10 minutes on {{ $labels.instance }}."
|
||||
|
||||
- alert: NodeDiskSpaceLow
|
||||
expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"}) * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk space low on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is above 85% full."
|
||||
|
||||
- alert: NodeDiskSpaceCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"}) * 100 > 95
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Disk space CRITICAL on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is above 95% full."
|
||||
|
||||
- alert: NodeHighMemory
|
||||
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage on {{ $labels.instance }}"
|
||||
description: "Memory usage above 90% on {{ $labels.instance }}."
|
||||
|
||||
- alert: NodeSwapUsage
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 80)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High swap usage on {{ $labels.instance }}"
|
||||
description: "Swap usage above 80% on {{ $labels.instance }}."
|
||||
|
||||
# === PROXMOX ===
|
||||
- name: proxmox_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: PVENodeDown
|
||||
expr: pve_up{id=~"node/.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox node {{ $labels.id }} is down"
|
||||
description: "PVE node {{ $labels.id }} is unreachable."
|
||||
|
||||
- alert: PVEHAServiceError
|
||||
expr: pve_ha_state{state=~"error|fence|freeze|gone"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PVE HA error: {{ $labels.id }} state={{ $labels.state }}"
|
||||
description: "HA-managed resource {{ $labels.id }} is in {{ $labels.state }} state."
|
||||
|
||||
- alert: PVEHighCPU
|
||||
expr: pve_cpu_usage_ratio > 0.90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PVE {{ $labels.id }} high CPU"
|
||||
description: "PVE guest {{ $labels.id }} CPU usage above 90% for 10 minutes."
|
||||
|
||||
- alert: PVEHighMemory
|
||||
expr: pve_memory_usage_bytes / pve_memory_size_bytes > 0.90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PVE {{ $labels.id }} high memory"
|
||||
description: "PVE guest {{ $labels.id }} memory usage above 90%."
|
||||
|
||||
# === CEPH ===
|
||||
- name: ceph_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: CephHealthWarning
|
||||
expr: ceph_health_status == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph cluster HEALTH_WARN"
|
||||
description: "Ceph cluster is in HEALTH_WARN state for 5 minutes."
|
||||
|
||||
- alert: CephHealthCritical
|
||||
expr: ceph_health_status == 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Ceph cluster HEALTH_ERR"
|
||||
description: "Ceph cluster is in HEALTH_ERR state!"
|
||||
|
||||
- alert: CephOSDDown
|
||||
expr: ceph_osd_down > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph OSDs down: {{ $value }}"
|
||||
description: "{{ $value }} Ceph OSD(s) are down."
|
||||
|
||||
- alert: CephPoolNearFull
|
||||
expr: ceph_pool_percent_used > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph pool {{ $labels.pool }} near full"
|
||||
description: "Ceph pool {{ $labels.pool }} is {{ $value }}% full."
|
||||
|
||||
# === GALERA ===
|
||||
- name: galera_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: GaleraNodeDown
|
||||
expr: mysql_up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Galera node {{ $labels.instance }} down"
|
||||
description: "MySQL/Galera node {{ $labels.instance }} is not responding."
|
||||
|
||||
- alert: GaleraClusterSizeReduced
|
||||
expr: mysql_global_status_wsrep_cluster_size < 3
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Galera cluster size reduced to {{ $value }}"
|
||||
description: "Galera cluster has only {{ $value }} node(s) — expected 3."
|
||||
|
||||
- alert: GaleraNodeNotSynced
|
||||
expr: mysql_global_status_wsrep_local_state != 4
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Galera node {{ $labels.instance }} not synced"
|
||||
description: "Galera node {{ $labels.instance }} is not in Synced state."
|
||||
|
||||
# === BLACKBOX ===
|
||||
- name: blackbox_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: HostUnreachableICMP
|
||||
expr: probe_success{job="blackbox_icmp"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Host {{ $labels.instance }} unreachable (ICMP)"
|
||||
description: "{{ $labels.instance }} not responding to ping for 2 minutes."
|
||||
|
||||
- alert: HTTPEndpointDown
|
||||
expr: probe_success{job="blackbox_http"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "HTTP endpoint {{ $labels.instance }} down"
|
||||
description: "{{ $labels.instance }} not returning 2xx for 2 minutes."
|
||||
|
||||
- alert: HTTPCertExpiringSoon
|
||||
expr: probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time() < 86400 * 14
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SSL cert expiring soon for {{ $labels.instance }}"
|
||||
description: "SSL certificate for {{ $labels.instance }} expires in less than 14 days."
|
||||
@@ -0,0 +1,82 @@
|
||||
# Working docker-compose.yml for monitoring stack on CT 141
|
||||
# Deployed 2026-07-05. Contains fixes for all pitfalls encountered.
|
||||
#
|
||||
# Fixes applied vs. initial draft:
|
||||
# - prometheus: user: root (avoids permission denied on config files)
|
||||
# - pve-exporter: config mounted to /etc/prometheus/pve.yml (hardcoded path)
|
||||
# - telegram-bridge: custom Flask app, not third-party image (none reliable found)
|
||||
# - All config dirs: chmod 644 on host before compose up
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
container_name: prometheus
|
||||
restart: unless-stopped
|
||||
user: root # FIX: avoid permission denied on config
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/rules:/etc/prometheus/rules:ro
|
||||
- prometheus-data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana-oss:11.5.2
|
||||
container_name: grafana
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=changeme
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
ports:
|
||||
- "3000:3000"
|
||||
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.27.0
|
||||
container_name: alertmanager
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
ports:
|
||||
- "9093:9093"
|
||||
|
||||
blackbox:
|
||||
image: prom/blackbox-exporter:v0.26.0
|
||||
container_name: blackbox
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
|
||||
ports:
|
||||
- "9115:9115"
|
||||
|
||||
pve-exporter:
|
||||
image: prompve/prometheus-pve-exporter:latest
|
||||
container_name: pve-exporter
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PVE_EXPORTER_CONFIG=/etc/prometheus/pve.yml
|
||||
volumes:
|
||||
# FIX: mount to /etc/prometheus/pve.yml — binary hardcodes this path
|
||||
- ./pve-exporter/pve-exporter.yml:/etc/prometheus/pve.yml:ro
|
||||
ports:
|
||||
- "9221:9221"
|
||||
|
||||
telegram-bridge:
|
||||
build: ./telegram-bridge
|
||||
container_name: telegram-bridge
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
# ⚠️ .env must contain TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID.
|
||||
# If using inline environment: instead, do NOT leave values empty —
|
||||
# the bridge will accept webhooks (HTTP 200) but silently fail to
|
||||
# send to Telegram. Check `docker logs telegram-bridge` for
|
||||
# "Chat ID:" showing empty if misconfigured.
|
||||
ports:
|
||||
- "9099:9099"
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
@@ -0,0 +1,12 @@
|
||||
# NUT ups.conf — NUT master host (e.g. 10.0.30.100)
|
||||
# Place at /etc/nut/ups.conf on the NUT master host.
|
||||
#
|
||||
# Driver section for APC Back-UPS ES 700G (USB-connected).
|
||||
# Adjust driver/port/vendorid/productid for other UPS models.
|
||||
|
||||
[apc]
|
||||
driver = usbhid-ups
|
||||
port = auto
|
||||
pollfreq = 30
|
||||
pollinterval = 2
|
||||
desc = "APC Back-UPS ES 700G"
|
||||
@@ -0,0 +1,15 @@
|
||||
# NUT upsd.users — NUT master host (e.g. 10.0.30.100)
|
||||
# Place at /etc/nut/upsd.users on the NUT master host.
|
||||
# Replace <NUT_MASTER_PASSWORD> and <NUT_SLAVE_PASSWORD> before deploying.
|
||||
#
|
||||
# Two users required:
|
||||
# upsadmin — master upsmon on the NUT host itself
|
||||
# upsslave — slave upsmon on all Proxmox nodes
|
||||
|
||||
[upsadmin]
|
||||
password = <NUT_MASTER_PASSWORD>
|
||||
upsmon master
|
||||
|
||||
[upsslave]
|
||||
password = <NUT_SLAVE_PASSWORD>
|
||||
upsmon slave
|
||||
@@ -0,0 +1,30 @@
|
||||
# NUT master upsmon.conf — UPS-connected host (e.g. 10.0.30.100)
|
||||
# Place at /etc/nut/upsmon.conf on the NUT master host.
|
||||
# Replace <NUT_MASTER_PASSWORD> before deploying.
|
||||
#
|
||||
# Master mode: monitors the local UPS directly via upsd.
|
||||
# Uses upssched as NOTIFYCMD for timed FSD (30s on battery → force shutdown).
|
||||
# Master shuts down LAST — slaves receive FSD and shut down first.
|
||||
|
||||
MONITOR apc@localhost 1 upsadmin <NUT_MASTER_PASSWORD> upsmon master
|
||||
|
||||
MINSUPPLIES 1
|
||||
SHUTDOWNCMD "/sbin/shutdown -h now"
|
||||
NOTIFYCMD /usr/sbin/upssched
|
||||
POLLFREQ 5
|
||||
POLLFREQALERT 5
|
||||
HOSTSYNC 15
|
||||
DEADTIME 15
|
||||
POWERDOWNFLAG "/etc/killpower"
|
||||
OFFDURATION 30
|
||||
RBWARNTIME 43200
|
||||
NOCOMMWARNTIME 300
|
||||
FINALDELAY 5
|
||||
|
||||
NOTIFYFLAG ONLINE SYSLOG+WALL
|
||||
NOTIFYFLAG ONBATT SYSLOG+WALL
|
||||
NOTIFYFLAG LOWBATT SYSLOG+WALL
|
||||
NOTIFYFLAG FSD SYSLOG+WALL
|
||||
NOTIFYFLAG SHUTDOWN SYSLOG+WALL
|
||||
NOTIFYFLAG COMMOK SYSLOG
|
||||
NOTIFYFLAG COMMBAD SYSLOG+WALL
|
||||
@@ -0,0 +1,29 @@
|
||||
# NUT slave upsmon.conf — Proxmox nodes
|
||||
# Place at /etc/nut/upsmon.conf on each Proxmox node.
|
||||
# Replace <NUT_MASTER_IP>, <NUT_SLAVE_PASSWORD> before deploying.
|
||||
#
|
||||
# This configures the node as a NUT slave that listens for FSD signals
|
||||
# from the NUT master. On FSD, it runs the shutdown script which
|
||||
# stops all VMs/CTs gracefully, then powers off the node.
|
||||
|
||||
MONITOR apc@<NUT_MASTER_IP> 1 upsslave <NUT_SLAVE_PASSWORD> slave
|
||||
|
||||
MINSUPPLIES 1
|
||||
SHUTDOWNCMD "/usr/local/bin/ups-shutdown.sh"
|
||||
POLLFREQ 5
|
||||
POLLFREQALERT 5
|
||||
HOSTSYNC 15
|
||||
DEADTIME 15
|
||||
POWERDOWNFLAG "/etc/killpower"
|
||||
OFFDURATION 30
|
||||
RBWARNTIME 43200
|
||||
NOCOMMWARNTIME 300
|
||||
FINALDELAY 5
|
||||
|
||||
NOTIFYFLAG ONLINE SYSLOG+WALL
|
||||
NOTIFYFLAG ONBATT SYSLOG+WALL
|
||||
NOTIFYFLAG LOWBATT SYSLOG+WALL
|
||||
NOTIFYFLAG FSD SYSLOG+WALL
|
||||
NOTIFYFLAG SHUTDOWN SYSLOG+WALL
|
||||
NOTIFYFLAG COMMOK SYSLOG
|
||||
NOTIFYFLAG COMMBAD SYSLOG+WALL
|
||||
@@ -0,0 +1,18 @@
|
||||
# NUT upssched.conf — NUT master host (e.g. 10.0.30.100)
|
||||
# Place at /etc/nut/upssched.conf on the NUT master host.
|
||||
#
|
||||
# Triggers FSD after UPS has been on battery for 30 seconds.
|
||||
# If power returns before 30s, the timer is cancelled.
|
||||
# Also triggers on communication loss (COMMBAD) as a safety net.
|
||||
#
|
||||
# Requires: mkdir -p /run/nut && chown nut:nut /run/nut
|
||||
# Requires: CMDSCRIPT at /etc/nut/upssched-cmd (see templates/upssched-cmd.sh)
|
||||
|
||||
CMDSCRIPT /etc/nut/upssched-cmd
|
||||
PIPEFN /run/nut/upssched.pipe
|
||||
LOCKFN /run/nut/upssched.lock
|
||||
|
||||
AT ONBATT * START-TIMER earlyshutdown 30
|
||||
AT ONLINE * CANCEL-TIMER earlyshutdown
|
||||
AT COMMOK * CANCEL-TIMER earlyshutdown
|
||||
AT COMMBAD * START-TIMER earlyshutdown 30
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
# PBS setup script for OpenStack VM (Ubuntu 24.04)
|
||||
# Run on the VM via: sudo bash /tmp/pbs-setup.sh
|
||||
# Usage: PBS_ADMIN_PASSWORD="yourpass" sudo -E bash /tmp/pbs-setup.sh
|
||||
#
|
||||
# Prerequisites: /dev/sdb is the data volume (attached Cinder volume)
|
||||
set -e
|
||||
|
||||
echo "=== Format data volume ==="
|
||||
mke2fs -t ext4 /dev/sdb
|
||||
|
||||
echo "=== Mount ==="
|
||||
mkdir -p /mnt/datastore/offsite
|
||||
mount /dev/sdb /mnt/datastore/offsite
|
||||
echo "/dev/sdb /mnt/datastore/offsite ext4 defaults,noatime 0 2" >> /etc/fstab
|
||||
df -h /mnt/datastore/offsite
|
||||
|
||||
echo "=== Add Proxmox repo ==="
|
||||
echo "deb http://download.proxmox.com/debian/pbs bookworm pbstest" > /etc/apt/sources.list.d/pbs.list
|
||||
wget -q http://download.proxmox.com/debian/proxmox-release-bookworm.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg
|
||||
apt-get update -qq 2>&1 | tail -5
|
||||
|
||||
echo "=== Install PBS ==="
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y proxmox-backup-server 2>&1 | tail -20
|
||||
|
||||
echo "=== Create datastore ==="
|
||||
proxmox-backup-manager datastore create offsite /mnt/datastore/offsite
|
||||
|
||||
echo "=== Create admin user ==="
|
||||
PBS_ADMIN_PASSWORD="${PBS_ADMIN_PASSWORD:-changeme}"
|
||||
proxmox-backup-manager user create admin@pbs --password "$PBS_ADMIN_PASSWORD"
|
||||
|
||||
echo "=== Grant admin ACL ==="
|
||||
proxmox-backup-manager acl update / Admin --auth-id admin@pbs
|
||||
|
||||
echo "=== Show fingerprint ==="
|
||||
openssl x509 -in /etc/proxmox-backup/proxy.pem -noout -fingerprint -sha256
|
||||
|
||||
echo "=== Done ==="
|
||||
@@ -0,0 +1,24 @@
|
||||
[Unit]
|
||||
Description=ComfyUI
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/ComfyUI
|
||||
ExecStart=/usr/bin/python3 main.py --listen 0.0.0.0 --port 8188
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
# GPU device selection for ROCm
|
||||
Environment=HIP_VISIBLE_DEVICES=0
|
||||
|
||||
# PATH must include ROCm binaries for rocminfo/rocm-smi calls from code
|
||||
Environment=PATH=/opt/rocm/bin:/usr/local/bin:/usr/bin:/bin
|
||||
|
||||
# LD_LIBRARY_PATH must include torch's native .so directory so torchaudio
|
||||
# and other extensions can find libtorch.so, libtorch_cpu.so etc.
|
||||
Environment=LD_LIBRARY_PATH=/usr/local/lib/python3.12/dist-packages/torch/lib:/opt/rocm/lib
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Alertmanager → Telegram bridge.
|
||||
Receives Alertmanager webhooks at /alert and forwards formatted messages
|
||||
to Telegram via Bot API. Also has /health endpoint for monitoring.
|
||||
|
||||
Environment variables:
|
||||
TELEGRAM_BOT_TOKEN — Bot API token
|
||||
TELEGRAM_CHAT_ID — Target chat ID (user or group)
|
||||
|
||||
Deploy via Docker: see templates/monitoring-docker-compose.yml
|
||||
Dockerfile: FROM python:3.12-slim; pip install flask requests; CMD python bridge.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
from flask import Flask, request, jsonify
|
||||
|
||||
BOT_TOKEN = os.environ.get("TELEGRAM_BOT_TOKEN", "")
|
||||
CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "")
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
def send_telegram(message: str):
|
||||
"""Send a message via Telegram Bot API."""
|
||||
url = f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage"
|
||||
payload = {
|
||||
"chat_id": CHAT_ID,
|
||||
"text": message,
|
||||
"parse_mode": "Markdown",
|
||||
"disable_web_page_preview": True,
|
||||
}
|
||||
try:
|
||||
resp = requests.post(url, json=payload, timeout=10)
|
||||
return resp.status_code == 200
|
||||
except Exception as e:
|
||||
print(f"Telegram send error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def format_alert(alert: dict) -> str:
|
||||
"""Format a single Alertmanager alert as a Markdown message."""
|
||||
status = alert.get("status", "unknown")
|
||||
labels = alert.get("labels", {})
|
||||
annotations = alert.get("annotations", {})
|
||||
|
||||
icon = "🔴" if status == "firing" else "🟢"
|
||||
|
||||
lines = [
|
||||
f"{icon} *{labels.get('alertname', 'Unknown')}* — `{labels.get('instance', '')}`",
|
||||
f"*Severity:* {labels.get('severity', '?')}",
|
||||
f"*Description:* {annotations.get('description', 'N/A')}",
|
||||
]
|
||||
if annotations.get("summary"):
|
||||
lines.append(f"*Summary:* {annotations['summary']}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@app.route("/alert", methods=["POST"])
|
||||
def receive_alert():
|
||||
"""Receive Alertmanager webhook and forward to Telegram."""
|
||||
data = request.get_json(force=True, silent=True)
|
||||
if not data:
|
||||
return jsonify({"error": "invalid JSON"}), 400
|
||||
|
||||
alerts = data.get("alerts", [])
|
||||
if not alerts:
|
||||
return jsonify({"status": "no alerts"}), 200
|
||||
|
||||
for alert in alerts:
|
||||
message = format_alert(alert)
|
||||
send_telegram(message)
|
||||
|
||||
return jsonify({"status": "sent", "count": len(alerts)}), 200
|
||||
|
||||
|
||||
@app.route("/pve", methods=["POST"])
|
||||
def pve_notification():
|
||||
"""Handle PVE webhook notifications (different format from Alertmanager).
|
||||
|
||||
PVE sends POST with JSON containing title/message/severity/type fields.
|
||||
Register in PVE via:
|
||||
pvesh create /cluster/notifications/endpoints/webhook \
|
||||
--name telegram-bridge \
|
||||
--url http://<monitoring-ct-ip>:9099/pve \
|
||||
--method POST
|
||||
Then test with:
|
||||
pvesh create /cluster/notifications/test telegram-bridge
|
||||
"""
|
||||
data = request.get_json(silent=True)
|
||||
if not data:
|
||||
data = request.form.to_dict()
|
||||
if not data:
|
||||
return jsonify({"error": "no data"}), 400
|
||||
|
||||
title = data.get("title", data.get("subject", ""))
|
||||
message = data.get("message", data.get("body", data.get("text", "")))
|
||||
severity = data.get("severity", data.get("priority", "info"))
|
||||
type_ = data.get("type", data.get("event", "notification"))
|
||||
|
||||
emoji = {"critical": "🔴", "warning": "🟡", "info": "🔵"}.get(severity, "🔵")
|
||||
msg = f"{emoji} *PVE: {title or type_}*"
|
||||
if message:
|
||||
if len(message) > 3500:
|
||||
message = message[:3500] + "\n...(truncated)"
|
||||
msg += f"\n\n{message}"
|
||||
|
||||
ok = send_telegram(msg)
|
||||
return jsonify({"status": "sent" if ok else "error"})
|
||||
|
||||
|
||||
@app.route("/health")
|
||||
def health():
|
||||
return jsonify({"status": "ok"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host="0.0.0.0", port=9099)
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# NUT-triggered controlled shutdown for Proxmox nodes
|
||||
# Place at /usr/local/bin/ups-shutdown.sh on each Proxmox node.
|
||||
# chmod 755 /usr/local/bin/ups-shutdown.sh
|
||||
#
|
||||
# Called by upsmon SHUTDOWNCMD when FSD is received from NUT master.
|
||||
# Stops all running VMs and CTs gracefully (60s timeout, then force),
|
||||
# waits for all to complete, then powers off the node.
|
||||
|
||||
logger -t nut-shutdown "Starting controlled guest shutdown on $(hostname)"
|
||||
|
||||
# Stop all running VMs gracefully
|
||||
for vmid in $(qm list --running 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
logger -t nut-shutdown "Stopping VM $vmid"
|
||||
qm shutdown "$vmid" --timeout 60 --forceStop 1 &
|
||||
done
|
||||
|
||||
# Stop all running CTs gracefully
|
||||
for ctid in $(pct list --running 2>/dev/null | awk 'NR>1{print $1}'); do
|
||||
logger -t nut-shutdown "Stopping CT $ctid"
|
||||
pct shutdown "$ctid" --timeout 60 --forceStop 1 &
|
||||
done
|
||||
|
||||
# Wait for all guest shutdowns to complete
|
||||
wait
|
||||
|
||||
logger -t nut-shutdown "All guests stopped, shutting down node $(hostname)"
|
||||
sleep 3
|
||||
shutdown -h now
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
# upssched-cmd — executed by upssched when a timer fires
|
||||
# Place at /etc/nut/upssched-cmd on the NUT master host.
|
||||
# chmod 755 /etc/nut/upssched-cmd
|
||||
#
|
||||
# When the earlyshutdown timer fires (UPS on battery for 30s),
|
||||
# this broadcasts FSD to all connected upsmon slaves.
|
||||
|
||||
case "$1" in
|
||||
earlyshutdown)
|
||||
logger -t upssched "UPS on battery for 30s — triggering FSD to all clients"
|
||||
/sbin/upsmon -c fsd
|
||||
;;
|
||||
*)
|
||||
logger -t upssched "Unknown upssched command: $1"
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user