Merge branch 'monitoring-resolved-alerts' into 'main'

Send resolved alerts

See merge request g4st3r/srv-docker!13

monitoring/alertmanager/alertmanager.yml

@@ -2,26 +2,14 @@ global:
   resolve_timeout: 5m
 
 route:
-  receiver: telegram
+  receiver: telegram-relay
   group_by: ['alertname', 'instance']
   group_wait: 30s
   group_interval: 5m
   repeat_interval: 6h
 
 receivers:
-  - name: telegram
+  - name: telegram-relay
-    telegram_configs:
+    webhook_configs:
-      - bot_token: '7896670660:AAGtfYPY47GfEhFLQeAR5oRGodV7Y6qa_LY'
+      - url: http://telegram-relay:8088/alert
-        chat_id: -1003779247128
-        message: |-
-          {{ range .Alerts }}
-          {{ if eq .Labels.severity "critical" }}🔴{{ else }}⚠️{{ end }} *{{ .Labels.alertname }}* ({{ .Labels.severity }})
-          {{ .Annotations.summary }}
-          {{ .Annotations.description }}
-          Instance: {{ .Labels.instance }}
-          Details: {{ .GeneratorURL }}
-          {{ end }}
-        parse_mode: Markdown
-        disable_notifications: false
-        api_url: 'https://api.telegram.org'
         send_resolved: true

monitoring/docker-compose.yml

@@ -33,6 +33,9 @@ services:
       - GF_SECURITY_ADMIN_USER=admin
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000
+      - GF_RENDERING_CONCURRENT_RENDER_REQUEST_LIMIT=5
     volumes:
       - grafana-data:/var/lib/grafana
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
@@ -41,6 +44,15 @@ services:
       - "3000:3000"
     depends_on:
       - prometheus
+      - grafana-image-renderer
+
+  grafana-image-renderer:
+    image: grafana/grafana-image-renderer:3.11.0
+    restart: unless-stopped
+    environment:
+      - ENABLE_METRICS=true
+    ports:
+      - "8081:8081"
 
   node-exporter:
     image: prom/node-exporter:v1.8.1
@@ -85,6 +97,21 @@ services:
     ports:
       - "9115:9115"
 
+  telegram-relay:
+    image: python:3.12-slim
+    restart: unless-stopped
+    env_file:
+      - ./.env
+    environment:
+      - GRAFANA_BASE=https://grafana.g4st3r.ru
+      - PROM_BASE=https://prometheus.g4st3r.ru
+    volumes:
+      - ./telegram-relay:/app:ro
+    working_dir: /app
+    command: ["/bin/sh","-lc","pip install -r requirements.txt && python app.py"]
+    ports:
+      - "8088:8088"
+
 volumes:
   prometheus-data:
   grafana-data:

monitoring/rules/base-alerts.yml

@@ -83,13 +83,13 @@ groups:
           description: "{{ $labels.mountpoint }} usage above 95%"
 
       - alert: DockerContainerDown
-        expr: time() - container_last_seen > 120
+        expr: time() - container_last_seen{image!="",container!=""} > 300
-        for: 2m
+        for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "Container down"
+          summary: "Контейнер недоступен"
-          description: "A container has not been seen for >2m"
+          description: "Контейнер не виден >5 минут"
 
       - alert: HttpEndpointDown
         expr: probe_success{job="blackbox-http"} == 0

monitoring/telegram-relay/app.py

@@ -0,0 +1,129 @@
+import os
+import requests
+from flask import Flask, request, jsonify
+
+app = Flask(__name__)
+
+TG_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
+TG_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID")
+TG_THREAD_ID = os.getenv("TELEGRAM_THREAD_ID")
+GRAFANA_TOKEN = os.getenv("GRAFANA_TOKEN")
+
+GRAFANA_BASE = os.getenv("GRAFANA_BASE", "https://grafana.g4st3r.ru")
+PROM_BASE = os.getenv("PROM_BASE", "https://prometheus.g4st3r.ru")
+
+
+def _emoji(sev):
+    return "🔴" if sev == "critical" else "⚠️"
+
+
+def _grafana_render_url(instance, alertname):
+    # map instance/alert to dashboard UID
+    if alertname == "DockerContainerDown":
+        uid = "affbyigwgb5dsb"  # cAdvisor — Containers
+    elif instance == "10.8.0.1:9100":
+        uid = "bffbyigsryj28c"  # Node — Reverse Proxy
+    elif instance == "node-exporter:9100":
+        uid = "affbyigukvw1sc"  # Node — Home Server
+    else:
+        return None
+    return f"{GRAFANA_BASE}/render/d/{uid}?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark"
+
+
+def _grafana_link(instance, alertname):
+    if alertname == "DockerContainerDown":
+        return f"{GRAFANA_BASE}/d/affbyigwgb5dsb"
+    if instance == "10.8.0.1:9100":
+        return f"{GRAFANA_BASE}/d/bffbyigsryj28c"
+    if instance == "node-exporter:9100":
+        return f"{GRAFANA_BASE}/d/affbyigukvw1sc"
+    return GRAFANA_BASE
+
+
+def _prom_link(gen_url):
+    if not gen_url:
+        return PROM_BASE
+    return gen_url.replace(gen_url.split("/")[0] + "//" + gen_url.split("/")[2], PROM_BASE)
+
+
+@app.route("/alert", methods=["POST"])
+def alert():
+    data = request.get_json(force=True)
+    alerts = data.get("alerts", [])
+    results = []
+    for a in alerts:
+        labels = a.get("labels", {})
+        status = a.get("status", {}).get("state", "firing")
+        is_resolved = status == "resolved"
+        ann = a.get("annotations", {})
+        sev = labels.get("severity", "warning")
+        alertname = labels.get("alertname", "Alert")
+        instance = labels.get("instance", "-")
+        summary = ann.get("summary", "")
+        desc = ann.get("description", "")
+
+        prom = _prom_link(a.get("generatorURL", ""))
+        graf = _grafana_link(instance, alertname)
+        render = _grafana_render_url(instance, alertname)
+
+        prefix = '✅' if is_resolved else _emoji(sev)
+        state_txt = 'восстановлено' if is_resolved else sev
+        caption = (
+            f"{prefix} {alertname} ({state_txt})\n"
+            f"{summary}\n{desc}\n"
+            f"Инстанс: {instance}"
+        )
+
+        # send photo if possible
+        if render and GRAFANA_TOKEN:
+            try:
+                img = requests.get(render, headers={"Authorization": f"Bearer {GRAFANA_TOKEN}"}, timeout=10)
+                if img.status_code == 200:
+                    files = {"photo": ("graph.png", img.content)}
+                    payload = {
+                        "chat_id": TG_CHAT_ID,
+                        "caption": caption,
+                    }
+                    if TG_THREAD_ID:
+                        payload["message_thread_id"] = TG_THREAD_ID
+                    requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendPhoto", data=payload, files=files, timeout=10)
+                else:
+                    raise Exception(f"render status {img.status_code}")
+            except Exception:
+                # fallback to text
+                payload = {
+                    "chat_id": TG_CHAT_ID,
+                    "text": caption,
+                }
+                if TG_THREAD_ID:
+                    payload["message_thread_id"] = TG_THREAD_ID
+                requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+        else:
+            payload = {
+                "chat_id": TG_CHAT_ID,
+                "text": caption,
+            }
+            if TG_THREAD_ID:
+                payload["message_thread_id"] = TG_THREAD_ID
+            requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+
+        # always send follow-up links
+        text_links = (
+            f"Ссылка (Prometheus): {prom}\n"
+            f"Ссылка (Grafana): {graf}"
+        )
+        payload = {
+            "chat_id": TG_CHAT_ID,
+            "text": text_links,
+            "disable_web_page_preview": True,
+        }
+        if TG_THREAD_ID:
+            payload["message_thread_id"] = TG_THREAD_ID
+        requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+        results.append({"alertname": alertname, "instance": instance})
+
+    return jsonify({"ok": True, "sent": results})
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=8088)

monitoring/telegram-relay/requirements.txt

@@ -0,0 +1,2 @@
+flask==3.0.2
+requests==2.31.0