Merge branch 'monitoring-telegram-relay' into 'main'

Add Telegram relay with photos See merge request g4st3r/srv-docker!12
2026-03-07 21:24:20 +00:00
parent e4e0c057b4 1a9d637d53
commit e6ec911405
4 changed files with 146 additions and 18 deletions
--- a/monitoring/alertmanager/alertmanager.yml
+++ b/monitoring/alertmanager/alertmanager.yml
@@ -2,28 +2,14 @@ global:
  resolve_timeout: 5m

 route:
-  receiver: telegram
+  receiver: telegram-relay
  group_by: ['alertname', 'instance']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 6h

 receivers:
-  - name: telegram
-    telegram_configs:
-      - bot_token: '7896670660:AAGtfYPY47GfEhFLQeAR5oRGodV7Y6qa_LY'
-        chat_id: -1003779247128
-        message: |-
-          {{ range .Alerts }}
-          {{ if eq .Labels.severity "critical" }}🔴{{ else }}⚠️{{ end }} *{{ .Labels.alertname }}* ({{ .Labels.severity }})
-          {{ .Annotations.summary }}
-          {{ .Annotations.description }}
-          Инстанс: {{ .Labels.instance }}
-          Ссылка (Prometheus): {{ reReplaceAll "http://[^/]+" "https://prometheus.g4st3r.ru" .GeneratorURL }}
-          Ссылка (Grafana): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/d/affbyigwgb5dsb{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/d/bffbyigsryj28c{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/d/affbyigukvw1sc{{ else }}https://grafana.g4st3r.ru{{ end }}
-          График (PNG): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/render/d/affbyigwgb5dsb?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/render/d/bffbyigsryj28c?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/render/d/affbyigukvw1sc?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else }}https://grafana.g4st3r.ru{{ end }}
-          {{ end }}
-        parse_mode: Markdown
-        disable_notifications: false
-        api_url: 'https://api.telegram.org'
+  - name: telegram-relay
+    webhook_configs:
+      - url: http://telegram-relay:8088/alert
        send_resolved: true
--- a/monitoring/docker-compose.yml
+++ b/monitoring/docker-compose.yml
@@ -97,6 +97,21 @@ services:
    ports:
      - "9115:9115"

+  telegram-relay:
+    image: python:3.12-slim
+    restart: unless-stopped
+    env_file:
+      - ./.env
+    environment:
+      - GRAFANA_BASE=https://grafana.g4st3r.ru
+      - PROM_BASE=https://prometheus.g4st3r.ru
+    volumes:
+      - ./telegram-relay:/app:ro
+    working_dir: /app
+    command: ["/bin/sh","-lc","pip install -r requirements.txt && python app.py"]
+    ports:
+      - "8088:8088"
+
 volumes:
  prometheus-data:
  grafana-data:
--- a/monitoring/telegram-relay/app.py
+++ b/monitoring/telegram-relay/app.py
@@ -0,0 +1,125 @@
+import os
+import requests
+from flask import Flask, request, jsonify
+
+app = Flask(__name__)
+
+TG_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
+TG_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID")
+TG_THREAD_ID = os.getenv("TELEGRAM_THREAD_ID")
+GRAFANA_TOKEN = os.getenv("GRAFANA_TOKEN")
+
+GRAFANA_BASE = os.getenv("GRAFANA_BASE", "https://grafana.g4st3r.ru")
+PROM_BASE = os.getenv("PROM_BASE", "https://prometheus.g4st3r.ru")
+
+
+def _emoji(sev):
+    return "🔴" if sev == "critical" else "⚠️"
+
+
+def _grafana_render_url(instance, alertname):
+    # map instance/alert to dashboard UID
+    if alertname == "DockerContainerDown":
+        uid = "affbyigwgb5dsb"  # cAdvisor — Containers
+    elif instance == "10.8.0.1:9100":
+        uid = "bffbyigsryj28c"  # Node — Reverse Proxy
+    elif instance == "node-exporter:9100":
+        uid = "affbyigukvw1sc"  # Node — Home Server
+    else:
+        return None
+    return f"{GRAFANA_BASE}/render/d/{uid}?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark"
+
+
+def _grafana_link(instance, alertname):
+    if alertname == "DockerContainerDown":
+        return f"{GRAFANA_BASE}/d/affbyigwgb5dsb"
+    if instance == "10.8.0.1:9100":
+        return f"{GRAFANA_BASE}/d/bffbyigsryj28c"
+    if instance == "node-exporter:9100":
+        return f"{GRAFANA_BASE}/d/affbyigukvw1sc"
+    return GRAFANA_BASE
+
+
+def _prom_link(gen_url):
+    if not gen_url:
+        return PROM_BASE
+    return gen_url.replace(gen_url.split("/")[0] + "//" + gen_url.split("/")[2], PROM_BASE)
+
+
+@app.route("/alert", methods=["POST"])
+def alert():
+    data = request.get_json(force=True)
+    alerts = data.get("alerts", [])
+    results = []
+    for a in alerts:
+        labels = a.get("labels", {})
+        ann = a.get("annotations", {})
+        sev = labels.get("severity", "warning")
+        alertname = labels.get("alertname", "Alert")
+        instance = labels.get("instance", "-")
+        summary = ann.get("summary", "")
+        desc = ann.get("description", "")
+
+        prom = _prom_link(a.get("generatorURL", ""))
+        graf = _grafana_link(instance, alertname)
+        render = _grafana_render_url(instance, alertname)
+
+        caption = (
+            f"{_emoji(sev)} {alertname} ({sev})\n"
+            f"{summary}\n{desc}\n"
+            f"Инстанс: {instance}"
+        )
+
+        # send photo if possible
+        if render and GRAFANA_TOKEN:
+            try:
+                img = requests.get(render, headers={"Authorization": f"Bearer {GRAFANA_TOKEN}"}, timeout=10)
+                if img.status_code == 200:
+                    files = {"photo": ("graph.png", img.content)}
+                    payload = {
+                        "chat_id": TG_CHAT_ID,
+                        "caption": caption,
+                    }
+                    if TG_THREAD_ID:
+                        payload["message_thread_id"] = TG_THREAD_ID
+                    requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendPhoto", data=payload, files=files, timeout=10)
+                else:
+                    raise Exception(f"render status {img.status_code}")
+            except Exception:
+                # fallback to text
+                payload = {
+                    "chat_id": TG_CHAT_ID,
+                    "text": caption,
+                }
+                if TG_THREAD_ID:
+                    payload["message_thread_id"] = TG_THREAD_ID
+                requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+        else:
+            payload = {
+                "chat_id": TG_CHAT_ID,
+                "text": caption,
+            }
+            if TG_THREAD_ID:
+                payload["message_thread_id"] = TG_THREAD_ID
+            requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+
+        # always send follow-up links
+        text_links = (
+            f"Ссылка (Prometheus): {prom}\n"
+            f"Ссылка (Grafana): {graf}"
+        )
+        payload = {
+            "chat_id": TG_CHAT_ID,
+            "text": text_links,
+            "disable_web_page_preview": True,
+        }
+        if TG_THREAD_ID:
+            payload["message_thread_id"] = TG_THREAD_ID
+        requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+        results.append({"alertname": alertname, "instance": instance})
+
+    return jsonify({"ok": True, "sent": results})
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=8088)
--- a/monitoring/telegram-relay/requirements.txt
+++ b/monitoring/telegram-relay/requirements.txt
@@ -0,0 +1,2 @@
+flask==3.0.2
+requests==2.31.0