Правила оповещений Prometheus

groups:
  - name: infra
    rules:
      - alert: InstanceDown
        expr: 'up == 0'
        for: 5m
        labels:
          severity: critical
          team: ops
        annotations:
          summary: 'Цель {{ $labels.instance }} не отвечает'
          description: 'Prometheus не может снять метрики с {{ $labels.job }} на {{ $labels.instance }} уже {{ $value }}.'

      - alert: Memory
        expr: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10'
        for: 10m
        labels:
          severity: warning
          team: ops
        annotations:
          summary: 'Память на {{ $labels.instance }} на исходе'
          description: 'Доступно {{ $value | printf "%.1f" }} % памяти.'

      - alert: Disk
        expr: 'node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} * 100 < 10'
        for: 10m
        labels:
          severity: critical
          team: ops
        annotations:
          summary: 'Диск {{ $labels.mountpoint }} на {{ $labels.instance }} заканчивается'
          description: 'Свободно {{ $value | printf "%.1f" }} %.'

      - alert: CertExpiry
        expr: 'probe_ssl_earliest_cert_expiry - time() < 14 * 24 * 3600'
        for: 1h
        labels:
          severity: warning
          team: ops
        annotations:
          summary: 'Сертификат {{ $labels.instance }} скоро истечёт'
          description: 'Осталось {{ $value | humanizeDuration }}.'

      - alert: SystemdUnit
        expr: 'node_systemd_unit_state{state="failed"} == 1'
        for: 5m
        labels:
          severity: critical
          team: ops
        annotations:
          summary: 'Юнит {{ $labels.name }} на {{ $labels.instance }} в состоянии failed'
          description: 'Посмотрите journalctl -u {{ $labels.name }}.'