Send resolved alerts with status

monitoring/alertmanager/alertmanager.yml

@@ -2,24 +2,14 @@ global:
   resolve_timeout: 5m
 
 route:
-  receiver: telegram
+  receiver: telegram-relay
   group_by: ['alertname', 'instance']
   group_wait: 30s
   group_interval: 5m
   repeat_interval: 6h
 
 receivers:
-  - name: telegram
-    telegram_configs:
-      - bot_token: '7896670660:AAGtfYPY47GfEhFLQeAR5oRGodV7Y6qa_LY'
-        chat_id: -1003779247128
-        message: |-
-          {{ range .Alerts }}
-          *{{ .Labels.alertname }}* ({{ .Labels.severity }})
-          {{ .Annotations.summary }}
-          {{ .Annotations.description }}
-          {{ end }}
-        parse_mode: Markdown
-        disable_notifications: false
-        api_url: 'https://api.telegram.org'
+  - name: telegram-relay
+    webhook_configs:
+      - url: http://telegram-relay:8088/alert
         send_resolved: true

monitoring/blackbox/blackbox.yml

@@ -0,0 +1,19 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      valid_status_codes: []
+      method: GET
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"
+  http_5xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      valid_status_codes: [500, 501, 502, 503, 504]
+      method: GET
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"

monitoring/docker-compose.yml

@@ -33,6 +33,9 @@ services:
       - GF_SECURITY_ADMIN_USER=admin
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000
+      - GF_RENDERING_CONCURRENT_RENDER_REQUEST_LIMIT=5
     volumes:
       - grafana-data:/var/lib/grafana
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
@@ -41,6 +44,15 @@ services:
       - "3000:3000"
     depends_on:
       - prometheus
+      - grafana-image-renderer
+
+  grafana-image-renderer:
+    image: grafana/grafana-image-renderer:3.11.0
+    restart: unless-stopped
+    environment:
+      - ENABLE_METRICS=true
+    ports:
+      - "8081:8081"
 
   node-exporter:
     image: prom/node-exporter:v1.8.1
@@ -77,6 +89,29 @@ services:
     ports:
       - "9633:9633"
 
+  blackbox-exporter:
+    image: prom/blackbox-exporter:v0.25.0
+    restart: unless-stopped
+    volumes:
+      - ./blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+    ports:
+      - "9115:9115"
+
+  telegram-relay:
+    image: python:3.12-slim
+    restart: unless-stopped
+    env_file:
+      - ./.env
+    environment:
+      - GRAFANA_BASE=https://grafana.g4st3r.ru
+      - PROM_BASE=https://prometheus.g4st3r.ru
+    volumes:
+      - ./telegram-relay:/app:ro
+    working_dir: /app
+    command: ["/bin/sh","-lc","pip install -r requirements.txt && python app.py"]
+    ports:
+      - "8088:8088"
+
 volumes:
   prometheus-data:
   grafana-data:

monitoring/prometheus/prometheus.yml

@@ -30,3 +30,55 @@ scrape_configs:
   - job_name: revproxy-node
     static_configs:
       - targets: ["10.8.0.1:9100"]
+
+  - job_name: blackbox-http
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - https://s3.g4st3r.ru
+          - https://minio.g4st3r.ru
+          - https://cloud.g4st3r.ru
+          - https://grafana.g4st3r.ru
+          - https://prometheus.g4st3r.ru
+          - https://alert.g4st3r.ru
+          - https://photos.g4st3r.ru
+          - https://home.g4st3r.ru
+          - https://gitlab.g4st3r.ru
+          - https://openclaw.g4st3r.ru
+          - https://git.g4st3r.ru
+          - https://bw.g4st3r.ru
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+
+  - job_name: blackbox-5xx
+    metrics_path: /probe
+    params:
+      module: [http_5xx]
+    static_configs:
+      - targets:
+          - https://s3.g4st3r.ru
+          - https://minio.g4st3r.ru
+          - https://cloud.g4st3r.ru
+          - https://grafana.g4st3r.ru
+          - https://prometheus.g4st3r.ru
+          - https://alert.g4st3r.ru
+          - https://photos.g4st3r.ru
+          - https://home.g4st3r.ru
+          - https://gitlab.g4st3r.ru
+          - https://openclaw.g4st3r.ru
+          - https://git.g4st3r.ru
+          - https://bw.g4st3r.ru
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115

monitoring/rules/base-alerts.yml

@@ -10,6 +10,15 @@ groups:
           summary: "{{ $labels.instance }} down"
           description: "{{ $labels.job }} on {{ $labels.instance }} is not responding"
 
+      - alert: NodeExporterDown
+        expr: up{job=~"node|revproxy-node"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Node exporter down on {{ $labels.instance }}"
+          description: "node_exporter is not responding"
+
       - alert: HighCPU
         expr: (1 - avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 0.90
         for: 10m
@@ -28,6 +37,24 @@ groups:
           summary: "High memory on {{ $labels.instance }}"
           description: "Memory usage > 90% for 10m"
 
+      - alert: MemAvailableLow
+        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.08
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Low available memory on {{ $labels.instance }}"
+          description: "MemAvailable < 8%"
+
+      - alert: SwapUsageHigh
+        expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) > 0.50
+        for: 20m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Swap usage high on {{ $labels.instance }}"
+          description: "Swap usage > 50%"
+
       - alert: DiskFull
         expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
         for: 15m
@@ -38,19 +65,46 @@ groups:
           description: "<10% free on {{ $labels.mountpoint }}"
 
       - alert: DiskUsageHigh
-        expr: (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})) > 0.85
+        expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"})) > 0.85
         for: 15m
         labels:
           severity: warning
         annotations:
           summary: "Disk usage >85% on {{ $labels.instance }}"
-          description: "Root filesystem usage above 85%"
+          description: "{{ $labels.mountpoint }} usage above 85%"
 
-      - alert: MemAvailableLow
-        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.08
-        for: 10m
+      - alert: DiskUsageCritical
+        expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"})) > 0.95
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Disk usage >95% on {{ $labels.instance }}"
+          description: "{{ $labels.mountpoint }} usage above 95%"
+
+      - alert: DockerContainerDown
+        expr: time() - container_last_seen{image!="",container!=""} > 300
+        for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "Low available memory on {{ $labels.instance }}"
-          description: "MemAvailable < 8%"
+          summary: "Контейнер недоступен"
+          description: "Контейнер не виден >5 минут"
+
+      - alert: HttpEndpointDown
+        expr: probe_success{job="blackbox-http"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "HTTP endpoint down: {{ $labels.instance }}"
+          description: "Blackbox probe failed"
+
+      - alert: HttpEndpoint5xx
+        expr: probe_success{job="blackbox-5xx"} == 1
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "HTTP 5xx detected: {{ $labels.instance }}"
+          description: "Endpoint returned 5xx"

monitoring/telegram-relay/app.py

@@ -0,0 +1,129 @@
+import os
+import requests
+from flask import Flask, request, jsonify
+
+app = Flask(__name__)
+
+TG_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN")
+TG_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID")
+TG_THREAD_ID = os.getenv("TELEGRAM_THREAD_ID")
+GRAFANA_TOKEN = os.getenv("GRAFANA_TOKEN")
+
+GRAFANA_BASE = os.getenv("GRAFANA_BASE", "https://grafana.g4st3r.ru")
+PROM_BASE = os.getenv("PROM_BASE", "https://prometheus.g4st3r.ru")
+
+
+def _emoji(sev):
+    return "🔴" if sev == "critical" else "⚠️"
+
+
+def _grafana_render_url(instance, alertname):
+    # map instance/alert to dashboard UID
+    if alertname == "DockerContainerDown":
+        uid = "affbyigwgb5dsb"  # cAdvisor — Containers
+    elif instance == "10.8.0.1:9100":
+        uid = "bffbyigsryj28c"  # Node — Reverse Proxy
+    elif instance == "node-exporter:9100":
+        uid = "affbyigukvw1sc"  # Node — Home Server
+    else:
+        return None
+    return f"{GRAFANA_BASE}/render/d/{uid}?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark"
+
+
+def _grafana_link(instance, alertname):
+    if alertname == "DockerContainerDown":
+        return f"{GRAFANA_BASE}/d/affbyigwgb5dsb"
+    if instance == "10.8.0.1:9100":
+        return f"{GRAFANA_BASE}/d/bffbyigsryj28c"
+    if instance == "node-exporter:9100":
+        return f"{GRAFANA_BASE}/d/affbyigukvw1sc"
+    return GRAFANA_BASE
+
+
+def _prom_link(gen_url):
+    if not gen_url:
+        return PROM_BASE
+    return gen_url.replace(gen_url.split("/")[0] + "//" + gen_url.split("/")[2], PROM_BASE)
+
+
+@app.route("/alert", methods=["POST"])
+def alert():
+    data = request.get_json(force=True)
+    alerts = data.get("alerts", [])
+    results = []
+    for a in alerts:
+        labels = a.get("labels", {})
+        status = a.get("status", {}).get("state", "firing")
+        is_resolved = status == "resolved"
+        ann = a.get("annotations", {})
+        sev = labels.get("severity", "warning")
+        alertname = labels.get("alertname", "Alert")
+        instance = labels.get("instance", "-")
+        summary = ann.get("summary", "")
+        desc = ann.get("description", "")
+
+        prom = _prom_link(a.get("generatorURL", ""))
+        graf = _grafana_link(instance, alertname)
+        render = _grafana_render_url(instance, alertname)
+
+        prefix = '✅' if is_resolved else _emoji(sev)
+        state_txt = 'восстановлено' if is_resolved else sev
+        caption = (
+            f"{prefix} {alertname} ({state_txt})\n"
+            f"{summary}\n{desc}\n"
+            f"Инстанс: {instance}"
+        )
+
+        # send photo if possible
+        if render and GRAFANA_TOKEN:
+            try:
+                img = requests.get(render, headers={"Authorization": f"Bearer {GRAFANA_TOKEN}"}, timeout=10)
+                if img.status_code == 200:
+                    files = {"photo": ("graph.png", img.content)}
+                    payload = {
+                        "chat_id": TG_CHAT_ID,
+                        "caption": caption,
+                    }
+                    if TG_THREAD_ID:
+                        payload["message_thread_id"] = TG_THREAD_ID
+                    requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendPhoto", data=payload, files=files, timeout=10)
+                else:
+                    raise Exception(f"render status {img.status_code}")
+            except Exception:
+                # fallback to text
+                payload = {
+                    "chat_id": TG_CHAT_ID,
+                    "text": caption,
+                }
+                if TG_THREAD_ID:
+                    payload["message_thread_id"] = TG_THREAD_ID
+                requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+        else:
+            payload = {
+                "chat_id": TG_CHAT_ID,
+                "text": caption,
+            }
+            if TG_THREAD_ID:
+                payload["message_thread_id"] = TG_THREAD_ID
+            requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+
+        # always send follow-up links
+        text_links = (
+            f"Ссылка (Prometheus): {prom}\n"
+            f"Ссылка (Grafana): {graf}"
+        )
+        payload = {
+            "chat_id": TG_CHAT_ID,
+            "text": text_links,
+            "disable_web_page_preview": True,
+        }
+        if TG_THREAD_ID:
+            payload["message_thread_id"] = TG_THREAD_ID
+        requests.post(f"https://api.telegram.org/bot{TG_TOKEN}/sendMessage", data=payload, timeout=10)
+        results.append({"alertname": alertname, "instance": instance})
+
+    return jsonify({"ok": True, "sent": results})
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=8088)

monitoring/telegram-relay/requirements.txt

@@ -0,0 +1,2 @@
+flask==3.0.2
+requests==2.31.0