Initial commit: Hermes Agent Skills collection
This commit is contained in:
Debian
@@ -0,0 +1,213 @@
|
||||
# Prometheus Alerting Rules for Proxmox VE + Ceph + Galera + Blackbox
|
||||
#
|
||||
# Copy to /opt/monitoring/prometheus/rules/alerting_rules.yml (or equivalent
|
||||
# rules dir mounted into the prometheus container) and reload:
|
||||
# docker exec prometheus wget -qO- --post-data= http://localhost:9090/-/reload
|
||||
#
|
||||
# CRITICAL PITFALLS (see references/alertmanager-webhook-setup.md):
|
||||
# - pve_up includes ALL guests (including intentionally stopped ones).
|
||||
# Filter with id=~"node/.*" for node-down alerts only.
|
||||
# - pve_ha_state has 15 states × hundreds of guests = 600+ series.
|
||||
# Only alert on state=~"error|fence|freeze|gone".
|
||||
# - After changing rules, restart alertmanager to clear stale cached alerts.
|
||||
|
||||
groups:
|
||||
# === NODE DOWN ===
|
||||
- name: node_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: NodeDown
|
||||
expr: up{job="node_exporter"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} is down"
|
||||
description: "{{ $labels.instance }} has been unreachable for 2 minutes."
|
||||
|
||||
- alert: NodeHighCPU
|
||||
expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{job="node_exporter",mode="idle"}[5m])) * 100) > 85
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High CPU on {{ $labels.instance }}"
|
||||
description: "CPU usage above 85% for 10 minutes on {{ $labels.instance }}."
|
||||
|
||||
- alert: NodeDiskSpaceLow
|
||||
expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"}) * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Disk space low on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is above 85% full."
|
||||
|
||||
- alert: NodeDiskSpaceCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"}) * 100 > 95
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Disk space CRITICAL on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is above 95% full."
|
||||
|
||||
- alert: NodeHighMemory
|
||||
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High memory usage on {{ $labels.instance }}"
|
||||
description: "Memory usage above 90% on {{ $labels.instance }}."
|
||||
|
||||
- alert: NodeSwapUsage
|
||||
expr: (node_memory_SwapTotal_bytes > 0) and ((1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 80)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High swap usage on {{ $labels.instance }}"
|
||||
description: "Swap usage above 80% on {{ $labels.instance }}."
|
||||
|
||||
# === PROXMOX ===
|
||||
- name: proxmox_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: PVENodeDown
|
||||
expr: pve_up{id=~"node/.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Proxmox node {{ $labels.id }} is down"
|
||||
description: "PVE node {{ $labels.id }} is unreachable."
|
||||
|
||||
- alert: PVEHAServiceError
|
||||
expr: pve_ha_state{state=~"error|fence|freeze|gone"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PVE HA error: {{ $labels.id }} state={{ $labels.state }}"
|
||||
description: "HA-managed resource {{ $labels.id }} is in {{ $labels.state }} state."
|
||||
|
||||
- alert: PVEHighCPU
|
||||
expr: pve_cpu_usage_ratio > 0.90
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PVE {{ $labels.id }} high CPU"
|
||||
description: "PVE guest {{ $labels.id }} CPU usage above 90% for 10 minutes."
|
||||
|
||||
- alert: PVEHighMemory
|
||||
expr: pve_memory_usage_bytes / pve_memory_size_bytes > 0.90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PVE {{ $labels.id }} high memory"
|
||||
description: "PVE guest {{ $labels.id }} memory usage above 90%."
|
||||
|
||||
# === CEPH ===
|
||||
- name: ceph_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: CephHealthWarning
|
||||
expr: ceph_health_status == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph cluster HEALTH_WARN"
|
||||
description: "Ceph cluster is in HEALTH_WARN state for 5 minutes."
|
||||
|
||||
- alert: CephHealthCritical
|
||||
expr: ceph_health_status == 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Ceph cluster HEALTH_ERR"
|
||||
description: "Ceph cluster is in HEALTH_ERR state!"
|
||||
|
||||
- alert: CephOSDDown
|
||||
expr: ceph_osd_down > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph OSDs down: {{ $value }}"
|
||||
description: "{{ $value }} Ceph OSD(s) are down."
|
||||
|
||||
- alert: CephPoolNearFull
|
||||
expr: ceph_pool_percent_used > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Ceph pool {{ $labels.pool }} near full"
|
||||
description: "Ceph pool {{ $labels.pool }} is {{ $value }}% full."
|
||||
|
||||
# === GALERA ===
|
||||
- name: galera_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: GaleraNodeDown
|
||||
expr: mysql_up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Galera node {{ $labels.instance }} down"
|
||||
description: "MySQL/Galera node {{ $labels.instance }} is not responding."
|
||||
|
||||
- alert: GaleraClusterSizeReduced
|
||||
expr: mysql_global_status_wsrep_cluster_size < 3
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Galera cluster size reduced to {{ $value }}"
|
||||
description: "Galera cluster has only {{ $value }} node(s) — expected 3."
|
||||
|
||||
- alert: GaleraNodeNotSynced
|
||||
expr: mysql_global_status_wsrep_local_state != 4
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Galera node {{ $labels.instance }} not synced"
|
||||
description: "Galera node {{ $labels.instance }} is not in Synced state."
|
||||
|
||||
# === BLACKBOX ===
|
||||
- name: blackbox_alerts
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: HostUnreachableICMP
|
||||
expr: probe_success{job="blackbox_icmp"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Host {{ $labels.instance }} unreachable (ICMP)"
|
||||
description: "{{ $labels.instance }} not responding to ping for 2 minutes."
|
||||
|
||||
- alert: HTTPEndpointDown
|
||||
expr: probe_success{job="blackbox_http"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "HTTP endpoint {{ $labels.instance }} down"
|
||||
description: "{{ $labels.instance }} not returning 2xx for 2 minutes."
|
||||
|
||||
- alert: HTTPCertExpiringSoon
|
||||
expr: probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time() < 86400 * 14
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "SSL cert expiring soon for {{ $labels.instance }}"
|
||||
description: "SSL certificate for {{ $labels.instance }} expires in less than 14 days."
|
||||
Reference in New Issue
Block a user