Translate alerts and tune container alert

monitoring/alertmanager/alertmanager.yml

@@ -15,9 +15,13 @@ receivers:
         chat_id: -1003779247128
         message: |-
           {{ range .Alerts }}
-          *{{ .Labels.alertname }}* ({{ .Labels.severity }})
+          {{ if eq .Labels.severity "critical" }}🔴{{ else }}⚠️{{ end }} *{{ .Labels.alertname }}* ({{ .Labels.severity }})
           {{ .Annotations.summary }}
           {{ .Annotations.description }}
+          Инстанс: {{ .Labels.instance }}
+          Ссылка (Prometheus): {{ reReplaceAll "http://[^/]+" "https://prometheus.g4st3r.ru" .GeneratorURL }}
+          Ссылка (Grafana): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/d/affbyigwgb5dsb{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/d/bffbyigsryj28c{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/d/affbyigukvw1sc{{ else }}https://grafana.g4st3r.ru{{ end }}
+          График (PNG): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/render/d/affbyigwgb5dsb?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/render/d/bffbyigsryj28c?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/render/d/affbyigukvw1sc?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else }}https://grafana.g4st3r.ru{{ end }}
           {{ end }}
         parse_mode: Markdown
         disable_notifications: false

monitoring/docker-compose.yml

@@ -33,6 +33,9 @@ services:
       - GF_SECURITY_ADMIN_USER=admin
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000
+      - GF_RENDERING_CONCURRENT_RENDER_REQUEST_LIMIT=5
     volumes:
       - grafana-data:/var/lib/grafana
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
@@ -41,6 +44,15 @@ services:
       - "3000:3000"
     depends_on:
       - prometheus
+      - grafana-image-renderer
+
+  grafana-image-renderer:
+    image: grafana/grafana-image-renderer:3.11.0
+    restart: unless-stopped
+    environment:
+      - ENABLE_METRICS=true
+    ports:
+      - "8081:8081"
 
   node-exporter:
     image: prom/node-exporter:v1.8.1

monitoring/rules/base-alerts.yml

@@ -47,13 +47,13 @@ groups:
           description: "MemAvailable < 8%"
 
       - alert: SwapUsageHigh
-        expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) > 0.30
+        expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) > 0.50
-        for: 15m
+        for: 20m
         labels:
           severity: warning
         annotations:
           summary: "Swap usage high on {{ $labels.instance }}"
-          description: "Swap usage > 30%"
+          description: "Swap usage > 50%"
 
       - alert: DiskFull
         expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
@@ -83,13 +83,13 @@ groups:
           description: "{{ $labels.mountpoint }} usage above 95%"
 
       - alert: DockerContainerDown
-        expr: time() - container_last_seen > 120
+        expr: time() - container_last_seen{image!="",container!=""} > 300
-        for: 2m
+        for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "Container down"
+          summary: "Контейнер недоступен"
-          description: "A container has not been seen for >2m"
+          description: "Контейнер не виден >5 минут"
 
       - alert: HttpEndpointDown
         expr: probe_success{job="blackbox-http"} == 0