130 lines
4.6 KiB
Python
130 lines
4.6 KiB
Python
import os
|
|
import requests
|
|
from flask import Flask, request, jsonify
|
|
|
|
app = Flask(__name__)
|
|
|
|
TG_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
|
|
TG_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID")
|
|
TG_THREAD_ID = os.getenv("TELEGRAM_THREAD_ID")
|
|
GRAFANA_TOKEN = os.getenv("GRAFANA_TOKEN")
|
|
|
|
GRAFANA_BASE = os.getenv("GRAFANA_BASE", "https://grafana.g4st3r.ru")
|
|
PROM_BASE = os.getenv("PROM_BASE", "https://prometheus.g4st3r.ru")
|
|
|
|
|
|
def _emoji(sev):
|
|
return "🔴" if sev == "critical" else "⚠️"
|
|
|
|
|
|
def _grafana_render_url(instance, alertname):
|
|
# map instance/alert to dashboard UID
|
|
if alertname == "DockerContainerDown":
|
|
uid = "affbyigwgb5dsb" # cAdvisor — Containers
|
|
elif instance == "10.8.0.1:9100":
|
|
uid = "bffbyigsryj28c" # Node — Reverse Proxy
|
|
elif instance == "node-exporter:9100":
|
|
uid = "affbyigukvw1sc" # Node — Home Server
|
|
else:
|
|
return None
|
|
return f"{GRAFANA_BASE}/render/d/{uid}?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark"
|
|
|
|
|
|
def _grafana_link(instance, alertname):
|
|
if alertname == "DockerContainerDown":
|
|
return f"{GRAFANA_BASE}/d/affbyigwgb5dsb"
|
|
if instance == "10.8.0.1:9100":
|
|
return f"{GRAFANA_BASE}/d/bffbyigsryj28c"
|
|
if instance == "node-exporter:9100":
|
|
return f"{GRAFANA_BASE}/d/affbyigukvw1sc"
|
|
return GRAFANA_BASE
|
|
|
|
|
|
def _prom_link(gen_url):
|
|
if not gen_url:
|
|
return PROM_BASE
|
|
return gen_url.replace(gen_url.split("/")[0] + "//" + gen_url.split("/")[2], PROM_BASE)
|
|
|
|
|
|
@app.route("/alert", methods=["POST"])
|
|
def alert():
|
|
data = request.get_json(force=True)
|
|
alerts = data.get("alerts", [])
|
|
results = []
|
|
for a in alerts:
|
|
labels = a.get("labels", {})
|
|
status = a.get("status", {}).get("state", "firing")
|
|
is_resolved = status == "resolved"
|
|
ann = a.get("annotations", {})
|
|
sev = labels.get("severity", "warning")
|
|
alertname = labels.get("alertname", "Alert")
|
|
instance = labels.get("instance", "-")
|
|
summary = ann.get("summary", "")
|
|
desc = ann.get("description", "")
|
|
|
|
prom = _prom_link(a.get("generatorURL", ""))
|
|
graf = _grafana_link(instance, alertname)
|
|
render = _grafana_render_url(instance, alertname)
|
|
|
|
prefix = '✅' if is_resolved else _emoji(sev)
|
|
state_txt = 'восстановлено' if is_resolved else sev
|
|
caption = (
|
|
f"{prefix} {alertname} ({state_txt})\n"
|
|
f"{summary}\n{desc}\n"
|
|
f"Инстанс: {instance}"
|
|
)
|
|
|
|
# send photo if possible
|
|
if render and GRAFANA_TOKEN:
|
|
try:
|
|
img = requests.get(render, headers={"Authorization": f"Bearer {GRAFANA_TOKEN}"}, timeout=10)
|
|
if img.status_code == 200:
|
|
files = {"photo": ("graph.png", img.content)}
|
|
payload = {
|
|
"chat_id": TG_CHAT_ID,
|
|
"caption": caption,
|
|
}
|
|
if TG_THREAD_ID:
|
|
payload["message_thread_id"] = TG_THREAD_ID
|
|
requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendPhoto", data=payload, files=files, timeout=10)
|
|
else:
|
|
raise Exception(f"render status {img.status_code}")
|
|
except Exception:
|
|
# fallback to text
|
|
payload = {
|
|
"chat_id": TG_CHAT_ID,
|
|
"text": caption,
|
|
}
|
|
if TG_THREAD_ID:
|
|
payload["message_thread_id"] = TG_THREAD_ID
|
|
requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
|
|
else:
|
|
payload = {
|
|
"chat_id": TG_CHAT_ID,
|
|
"text": caption,
|
|
}
|
|
if TG_THREAD_ID:
|
|
payload["message_thread_id"] = TG_THREAD_ID
|
|
requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
|
|
|
|
# always send follow-up links
|
|
text_links = (
|
|
f"Ссылка (Prometheus): {prom}\n"
|
|
f"Ссылка (Grafana): {graf}"
|
|
)
|
|
payload = {
|
|
"chat_id": TG_CHAT_ID,
|
|
"text": text_links,
|
|
"disable_web_page_preview": True,
|
|
}
|
|
if TG_THREAD_ID:
|
|
payload["message_thread_id"] = TG_THREAD_ID
|
|
requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
|
|
results.append({"alertname": alertname, "instance": instance})
|
|
|
|
return jsonify({"ok": True, "sent": results})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app.run(host="0.0.0.0", port=8088)
|