hermes-skills/devops/proxmox-ve-administration/templates/alerting_rules.yml

# Prometheus Alerting Rules for Proxmox VE + Ceph + Galera + Blackbox
#
# Copy to /opt/monitoring/prometheus/rules/alerting_rules.yml (or equivalent
# rules dir mounted into the prometheus container) and reload:
#   docker exec prometheus wget -qO- --post-data= http://localhost:9090/-/reload
#
# CRITICAL PITFALLS (see references/alertmanager-webhook-setup.md):
#   - pve_up includes ALL guests (including intentionally stopped ones).
#     Filter with id=~"node/.*" for node-down alerts only.
#   - pve_ha_state has 15 states × hundreds of guests = 600+ series.
#     Only alert on state=~"error|fence|freeze|gone".
#   - After changing rules, restart alertmanager to clear stale cached alerts.

groups:
  # === NODE DOWN ===
  - name: node_alerts
    interval: 30s
    rules:
      - alert: NodeDown
        expr: up{job="node_exporter"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Node {{ $labels.instance }} is down"
          description: "{{ $labels.instance }} has been unreachable for 2 minutes."

      - alert: NodeHighCPU
        expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{job="node_exporter",mode="idle"}[5m])) * 100) > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High CPU on {{ $labels.instance }}"
          description: "CPU usage above 85% for 10 minutes on {{ $labels.instance }}."

      - alert: NodeDiskSpaceLow
        expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"}) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Disk space low on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is above 85% full."

      - alert: NodeDiskSpaceCritical
        expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxc|overlay|squashfs"}) * 100 > 95
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Disk space CRITICAL on {{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is above 95% full."

      - alert: NodeHighMemory
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: "Memory usage above 90% on {{ $labels.instance }}."

      - alert: NodeSwapUsage
        expr: (node_memory_SwapTotal_bytes > 0) and ((1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 80)
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High swap usage on {{ $labels.instance }}"
          description: "Swap usage above 80% on {{ $labels.instance }}."

  # === PROXMOX ===
  - name: proxmox_alerts
    interval: 30s
    rules:
      - alert: PVENodeDown
        expr: pve_up{id=~"node/.*"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Proxmox node {{ $labels.id }} is down"
          description: "PVE node {{ $labels.id }} is unreachable."

      - alert: PVEHAServiceError
        expr: pve_ha_state{state=~"error|fence|freeze|gone"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "PVE HA error: {{ $labels.id }} state={{ $labels.state }}"
          description: "HA-managed resource {{ $labels.id }} is in {{ $labels.state }} state."

      - alert: PVEHighCPU
        expr: pve_cpu_usage_ratio > 0.90
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "PVE {{ $labels.id }} high CPU"
          description: "PVE guest {{ $labels.id }} CPU usage above 90% for 10 minutes."

      - alert: PVEHighMemory
        expr: pve_memory_usage_bytes / pve_memory_size_bytes > 0.90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "PVE {{ $labels.id }} high memory"
          description: "PVE guest {{ $labels.id }} memory usage above 90%."

  # === CEPH ===
  - name: ceph_alerts
    interval: 30s
    rules:
      - alert: CephHealthWarning
        expr: ceph_health_status == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Ceph cluster HEALTH_WARN"
          description: "Ceph cluster is in HEALTH_WARN state for 5 minutes."

      - alert: CephHealthCritical
        expr: ceph_health_status == 2
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Ceph cluster HEALTH_ERR"
          description: "Ceph cluster is in HEALTH_ERR state!"

      - alert: CephOSDDown
        expr: ceph_osd_down > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Ceph OSDs down: {{ $value }}"
          description: "{{ $value }} Ceph OSD(s) are down."

      - alert: CephPoolNearFull
        expr: ceph_pool_percent_used > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Ceph pool {{ $labels.pool }} near full"
          description: "Ceph pool {{ $labels.pool }} is {{ $value }}% full."

  # === GALERA ===
  - name: galera_alerts
    interval: 30s
    rules:
      - alert: GaleraNodeDown
        expr: mysql_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Galera node {{ $labels.instance }} down"
          description: "MySQL/Galera node {{ $labels.instance }} is not responding."

      - alert: GaleraClusterSizeReduced
        expr: mysql_global_status_wsrep_cluster_size < 3
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Galera cluster size reduced to {{ $value }}"
          description: "Galera cluster has only {{ $value }} node(s) — expected 3."

      - alert: GaleraNodeNotSynced
        expr: mysql_global_status_wsrep_local_state != 4
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Galera node {{ $labels.instance }} not synced"
          description: "Galera node {{ $labels.instance }} is not in Synced state."

  # === BLACKBOX ===
  - name: blackbox_alerts
    interval: 30s
    rules:
      - alert: HostUnreachableICMP
        expr: probe_success{job="blackbox_icmp"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Host {{ $labels.instance }} unreachable (ICMP)"
          description: "{{ $labels.instance }} not responding to ping for 2 minutes."

      - alert: HTTPEndpointDown
        expr: probe_success{job="blackbox_http"} == 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "HTTP endpoint {{ $labels.instance }} down"
          description: "{{ $labels.instance }} not returning 2xx for 2 minutes."

      - alert: HTTPCertExpiringSoon
        expr: probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time() < 86400 * 14
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "SSL cert expiring soon for {{ $labels.instance }}"
          description: "SSL certificate for {{ $labels.instance }} expires in less than 14 days."