Alert on all filesystems usage

monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,25 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: telegram
+  group_by: ['alertname', 'instance']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 6h
+
+receivers:
+  - name: telegram
+    telegram_configs:
+      - bot_token: '7896670660:AAGtfYPY47GfEhFLQeAR5oRGodV7Y6qa_LY'
+        chat_id: -1003779247128
+        message: |-
+          {{ range .Alerts }}
+          *{{ .Labels.alertname }}* ({{ .Labels.severity }})
+          {{ .Annotations.summary }}
+          {{ .Annotations.description }}
+          {{ end }}
+        parse_mode: Markdown
+        disable_notifications: false
+        api_url: 'https://api.telegram.org'
+        send_resolved: true

monitoring/docker-compose.yml

@@ -21,10 +21,8 @@ services:
     env_file:
       - ./.env
     volumes:
-      - ./alertmanager/alertmanager.tmpl.yml:/etc/alertmanager/alertmanager.tmpl.yml:ro
-      - ./alertmanager/render.sh:/render.sh:ro
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
       - alertmanager-data:/alertmanager
-    command: ["/render.sh"]
     ports:
       - "9093:9093"
 

monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -12,5 +12,6 @@ providers:
     type: file
     disableDeletion: false
     editable: true
+    allowUiUpdates: true
     options:
       path: /var/lib/grafana/dashboards

monitoring/prometheus/prometheus.yml

@@ -26,3 +26,7 @@ scrape_configs:
   - job_name: smartctl
     static_configs:
       - targets: ["smartctl-exporter:9633"]
+
+  - job_name: revproxy-node
+    static_configs:
+      - targets: ["10.8.0.1:9100"]

monitoring/rules/base-alerts.yml

@@ -10,6 +10,15 @@ groups:
           summary: "{{ $labels.instance }} down"
           description: "{{ $labels.job }} on {{ $labels.instance }} is not responding"
 
+      - alert: NodeExporterDown
+        expr: up{job=~"node|revproxy-node"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Node exporter down on {{ $labels.instance }}"
+          description: "node_exporter is not responding"
+
       - alert: HighCPU
         expr: (1 - avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 0.90
         for: 10m
@@ -28,6 +37,24 @@ groups:
           summary: "High memory on {{ $labels.instance }}"
           description: "Memory usage > 90% for 10m"
 
+      - alert: MemAvailableLow
+        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.08
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Low available memory on {{ $labels.instance }}"
+          description: "MemAvailable < 8%"
+
+      - alert: SwapUsageHigh
+        expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) > 0.30
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Swap usage high on {{ $labels.instance }}"
+          description: "Swap usage > 30%"
+
       - alert: DiskFull
         expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
         for: 15m
@@ -36,3 +63,30 @@ groups:
         annotations:
           summary: "Disk low on {{ $labels.instance }}"
           description: "<10% free on {{ $labels.mountpoint }}"
+
+      - alert: DiskUsageHigh
+        expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"})) > 0.85
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Disk usage >85% on {{ $labels.instance }}"
+          description: "{{ $labels.mountpoint }} usage above 85%"
+
+      - alert: DiskUsageCritical
+        expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"})) > 0.95
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Disk usage >95% on {{ $labels.instance }}"
+          description: "{{ $labels.mountpoint }} usage above 95%"
+
+      - alert: DockerContainerDown
+        expr: time() - container_last_seen > 120
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container down"
+          description: "A container has not been seen for >2m"