diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml index a2f1319..56e91ad 100644 --- a/monitoring/alertmanager/alertmanager.yml +++ b/monitoring/alertmanager/alertmanager.yml @@ -2,28 +2,14 @@ global: resolve_timeout: 5m route: - receiver: telegram + receiver: telegram-relay group_by: ['alertname', 'instance'] group_wait: 30s group_interval: 5m repeat_interval: 6h receivers: - - name: telegram - telegram_configs: - - bot_token: '7896670660:AAGtfYPY47GfEhFLQeAR5oRGodV7Y6qa_LY' - chat_id: -1003779247128 - message: |- - {{ range .Alerts }} - {{ if eq .Labels.severity "critical" }}🔴{{ else }}⚠️{{ end }} *{{ .Labels.alertname }}* ({{ .Labels.severity }}) - {{ .Annotations.summary }} - {{ .Annotations.description }} - Инстанс: {{ .Labels.instance }} - Ссылка (Prometheus): {{ reReplaceAll "http://[^/]+" "https://prometheus.g4st3r.ru" .GeneratorURL }} - Ссылка (Grafana): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/d/affbyigwgb5dsb{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/d/bffbyigsryj28c{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/d/affbyigukvw1sc{{ else }}https://grafana.g4st3r.ru{{ end }} - График (PNG): {{ if eq .Labels.alertname "DockerContainerDown" }}https://grafana.g4st3r.ru/render/d/affbyigwgb5dsb?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/render/d/bffbyigsryj28c?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/render/d/affbyigukvw1sc?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else }}https://grafana.g4st3r.ru{{ end }} - {{ end }} - parse_mode: Markdown - disable_notifications: false - api_url: 'https://api.telegram.org' + - name: telegram-relay + webhook_configs: + - url: http://telegram-relay:8088/alert send_resolved: true diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index fe9b927..27d0f07 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -97,6 +97,21 @@ services: ports: - "9115:9115" + telegram-relay: + image: python:3.12-slim + restart: unless-stopped + env_file: + - ./.env + environment: + - GRAFANA_BASE=https://grafana.g4st3r.ru + - PROM_BASE=https://prometheus.g4st3r.ru + volumes: + - ./telegram-relay:/app:ro + working_dir: /app + command: ["/bin/sh","-lc","pip install -r requirements.txt && python app.py"] + ports: + - "8088:8088" + volumes: prometheus-data: grafana-data: diff --git a/monitoring/telegram-relay/app.py b/monitoring/telegram-relay/app.py new file mode 100644 index 0000000..b30ca7a --- /dev/null +++ b/monitoring/telegram-relay/app.py @@ -0,0 +1,125 @@ +import os +import requests +from flask import Flask, request, jsonify + +app = Flask(__name__) + +TG_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN") +TG_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID") +TG_THREAD_ID = os.getenv("TELEGRAM_THREAD_ID") +GRAFANA_TOKEN = os.getenv("GRAFANA_TOKEN") + +GRAFANA_BASE = os.getenv("GRAFANA_BASE", "https://grafana.g4st3r.ru") +PROM_BASE = os.getenv("PROM_BASE", "https://prometheus.g4st3r.ru") + + +def _emoji(sev): + return "🔴" if sev == "critical" else "⚠️" + + +def _grafana_render_url(instance, alertname): + # map instance/alert to dashboard UID + if alertname == "DockerContainerDown": + uid = "affbyigwgb5dsb" # cAdvisor — Containers + elif instance == "10.8.0.1:9100": + uid = "bffbyigsryj28c" # Node — Reverse Proxy + elif instance == "node-exporter:9100": + uid = "affbyigukvw1sc" # Node — Home Server + else: + return None + return f"{GRAFANA_BASE}/render/d/{uid}?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark" + + +def _grafana_link(instance, alertname): + if alertname == "DockerContainerDown": + return f"{GRAFANA_BASE}/d/affbyigwgb5dsb" + if instance == "10.8.0.1:9100": + return f"{GRAFANA_BASE}/d/bffbyigsryj28c" + if instance == "node-exporter:9100": + return f"{GRAFANA_BASE}/d/affbyigukvw1sc" + return GRAFANA_BASE + + +def _prom_link(gen_url): + if not gen_url: + return PROM_BASE + return gen_url.replace(gen_url.split("/")[0] + "//" + gen_url.split("/")[2], PROM_BASE) + + +@app.route("/alert", methods=["POST"]) +def alert(): + data = request.get_json(force=True) + alerts = data.get("alerts", []) + results = [] + for a in alerts: + labels = a.get("labels", {}) + ann = a.get("annotations", {}) + sev = labels.get("severity", "warning") + alertname = labels.get("alertname", "Alert") + instance = labels.get("instance", "-") + summary = ann.get("summary", "") + desc = ann.get("description", "") + + prom = _prom_link(a.get("generatorURL", "")) + graf = _grafana_link(instance, alertname) + render = _grafana_render_url(instance, alertname) + + caption = ( + f"{_emoji(sev)} {alertname} ({sev})\n" + f"{summary}\n{desc}\n" + f"Инстанс: {instance}" + ) + + # send photo if possible + if render and GRAFANA_TOKEN: + try: + img = requests.get(render, headers={"Authorization": f"Bearer {GRAFANA_TOKEN}"}, timeout=10) + if img.status_code == 200: + files = {"photo": ("graph.png", img.content)} + payload = { + "chat_id": TG_CHAT_ID, + "caption": caption, + } + if TG_THREAD_ID: + payload["message_thread_id"] = TG_THREAD_ID + requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendPhoto", data=payload, files=files, timeout=10) + else: + raise Exception(f"render status {img.status_code}") + except Exception: + # fallback to text + payload = { + "chat_id": TG_CHAT_ID, + "text": caption, + } + if TG_THREAD_ID: + payload["message_thread_id"] = TG_THREAD_ID + requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10) + else: + payload = { + "chat_id": TG_CHAT_ID, + "text": caption, + } + if TG_THREAD_ID: + payload["message_thread_id"] = TG_THREAD_ID + requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10) + + # always send follow-up links + text_links = ( + f"Ссылка (Prometheus): {prom}\n" + f"Ссылка (Grafana): {graf}" + ) + payload = { + "chat_id": TG_CHAT_ID, + "text": text_links, + "disable_web_page_preview": True, + } + if TG_THREAD_ID: + payload["message_thread_id"] = TG_THREAD_ID + requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10) + results.append({"alertname": alertname, "instance": instance}) + + return jsonify({"ok": True, "sent": results}) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8088) diff --git a/monitoring/telegram-relay/requirements.txt b/monitoring/telegram-relay/requirements.txt new file mode 100644 index 0000000..bc0b9a9 --- /dev/null +++ b/monitoring/telegram-relay/requirements.txt @@ -0,0 +1,2 @@ +flask==3.0.2 +requests==2.31.0