Expand alert rules

monitoring/rules/base-alerts.yml

@@ -10,6 +10,15 @@ groups:
           summary: "{{ $labels.instance }} down"
           description: "{{ $labels.job }} on {{ $labels.instance }} is not responding"
 
+      - alert: NodeExporterDown
+        expr: up{job=~"node|revproxy-node"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Node exporter down on {{ $labels.instance }}"
+          description: "node_exporter is not responding"
+
       - alert: HighCPU
         expr: (1 - avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 0.90
         for: 10m
@@ -28,6 +37,24 @@ groups:
           summary: "High memory on {{ $labels.instance }}"
           description: "Memory usage > 90% for 10m"
 
+      - alert: MemAvailableLow
+        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.08
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Low available memory on {{ $labels.instance }}"
+          description: "MemAvailable < 8%"
+
+      - alert: SwapUsageHigh
+        expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) > 0.30
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Swap usage high on {{ $labels.instance }}"
+          description: "Swap usage > 30%"
+
       - alert: DiskFull
         expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
         for: 15m
@@ -46,11 +73,20 @@ groups:
           summary: "Disk usage >85% on {{ $labels.instance }}"
           description: "Root filesystem usage above 85%"
 
-      - alert: MemAvailableLow
+      - alert: DiskUsageCritical
-        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.08
+        expr: (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})) > 0.95
-        for: 10m
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Disk usage >95% on {{ $labels.instance }}"
+          description: "Root filesystem usage above 95%"
+
+      - alert: DockerContainerDown
+        expr: time() - container_last_seen > 120
+        for: 2m
         labels:
           severity: warning
         annotations:
-          summary: "Low available memory on {{ $labels.instance }}"
+          summary: "Container down"
-          description: "MemAvailable < 8%"
+          description: "A container has not been seen for >2m"