Translate alerts and tune container alert

monitoring/alertmanager/alertmanager.yml

@@ -18,8 +18,10 @@ receivers:
           {{ if eq .Labels.severity "critical" }}🔴{{ else }}⚠️{{ end }} *{{ .Labels.alertname }}* ({{ .Labels.severity }})
           {{ .Annotations.summary }}
           {{ .Annotations.description }}
-          Instance: {{ .Labels.instance }}
+          Инстанс: {{ .Labels.instance }}
-          Details: {{ .GeneratorURL }}
+          Ссылка (Prometheus): {{ reReplaceAll "http://[^/]+" "https://prometheus.g4st3r.ru" .GeneratorURL }}
+          Ссылка (Grafana): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/d/affbyigwgb5dsb{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/d/bffbyigsryj28c{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/d/affbyigukvw1sc{{ else }}https://grafana.g4st3r.ru{{ end }}
+          График (PNG): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/render/d/affbyigwgb5dsb?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/render/d/bffbyigsryj28c?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/render/d/affbyigukvw1sc?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else }}https://grafana.g4st3r.ru{{ end }}
           {{ end }}
         parse_mode: Markdown
         disable_notifications: false

monitoring/docker-compose.yml

@@ -33,6 +33,9 @@ services:
       - GF_SECURITY_ADMIN_USER=admin
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000
+      - GF_RENDERING_CONCURRENT_RENDER_REQUEST_LIMIT=5
     volumes:
       - grafana-data:/var/lib/grafana
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
@@ -41,6 +44,15 @@ services:
       - "3000:3000"
     depends_on:
       - prometheus
+      - grafana-image-renderer
+
+  grafana-image-renderer:
+    image: grafana/grafana-image-renderer:3.11.0
+    restart: unless-stopped
+    environment:
+      - ENABLE_METRICS=true
+    ports:
+      - "8081:8081"
 
   node-exporter:
     image: prom/node-exporter:v1.8.1

monitoring/rules/base-alerts.yml

@@ -83,13 +83,13 @@ groups:
           description: "{{ $labels.mountpoint }} usage above 95%"
 
       - alert: DockerContainerDown
-        expr: time() - container_last_seen > 120
+        expr: time() - container_last_seen{image!="",container!=""} > 300
-        for: 2m
+        for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "Container down"
+          summary: "Контейнер недоступен"
-          description: "A container has not been seen for >2m"
+          description: "Контейнер не виден >5 минут"
 
       - alert: HttpEndpointDown
         expr: probe_success{job="blackbox-http"} == 0