Add Grafana renderer and graph links

monitoring/alertmanager/alertmanager.yml

@@ -15,9 +15,13 @@ receivers:
         chat_id: -1003779247128
         message: |-
           {{ range .Alerts }}
-          *{{ .Labels.alertname }}* ({{ .Labels.severity }})
+          {{ if eq .Labels.severity "critical" }}🔴{{ else }}⚠️{{ end }} *{{ .Labels.alertname }}* ({{ .Labels.severity }})
           {{ .Annotations.summary }}
           {{ .Annotations.description }}
+          Instance: {{ .Labels.instance }}
+          Details (Prometheus): {{ reReplaceAll "http://[^/]+" "https://prometheus.g4st3r.ru" .GeneratorURL }}
+          Details (Grafana): {{ if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/d/bffbyigsryj28c{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/d/affbyigukvw1sc{{ else }}https://grafana.g4st3r.ru{{ end }}
+          Graph (PNG): {{ if eq .Labels.instance "10.8.0.1:9100" }}https://grafana.g4st3r.ru/render/d/bffbyigsryj28c?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else if eq .Labels.instance "node-exporter:9100" }}https://grafana.g4st3r.ru/render/d/affbyigukvw1sc?orgId=1&from=now-6h&to=now&width=1000&height=500&theme=dark{{ else }}https://grafana.g4st3r.ru{{ end }}
           {{ end }}
         parse_mode: Markdown
         disable_notifications: false

monitoring/blackbox/blackbox.yml

@@ -0,0 +1,19 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      valid_status_codes: []
+      method: GET
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"
+  http_5xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      valid_status_codes: [500, 501, 502, 503, 504]
+      method: GET
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"

monitoring/docker-compose.yml

@@ -33,6 +33,9 @@ services:
       - GF_SECURITY_ADMIN_USER=admin
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000
+      - GF_RENDERING_CONCURRENT_RENDER_REQUEST_LIMIT=5
     volumes:
       - grafana-data:/var/lib/grafana
       - ./grafana/provisioning:/etc/grafana/provisioning:ro
@@ -41,6 +44,15 @@ services:
       - "3000:3000"
     depends_on:
       - prometheus
+      - grafana-image-renderer
+
+  grafana-image-renderer:
+    image: grafana/grafana-image-renderer:3.11.0
+    restart: unless-stopped
+    environment:
+      - ENABLE_METRICS=true
+    ports:
+      - "8081:8081"
 
   node-exporter:
     image: prom/node-exporter:v1.8.1
@@ -77,6 +89,14 @@ services:
     ports:
       - "9633:9633"
 
+  blackbox-exporter:
+    image: prom/blackbox-exporter:v0.25.0
+    restart: unless-stopped
+    volumes:
+      - ./blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+    ports:
+      - "9115:9115"
+
 volumes:
   prometheus-data:
   grafana-data:

monitoring/prometheus/prometheus.yml

@@ -30,3 +30,55 @@ scrape_configs:
   - job_name: revproxy-node
     static_configs:
       - targets: ["10.8.0.1:9100"]
+
+  - job_name: blackbox-http
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - https://s3.g4st3r.ru
+          - https://minio.g4st3r.ru
+          - https://cloud.g4st3r.ru
+          - https://grafana.g4st3r.ru
+          - https://prometheus.g4st3r.ru
+          - https://alert.g4st3r.ru
+          - https://photos.g4st3r.ru
+          - https://home.g4st3r.ru
+          - https://gitlab.g4st3r.ru
+          - https://openclaw.g4st3r.ru
+          - https://git.g4st3r.ru
+          - https://bw.g4st3r.ru
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+
+  - job_name: blackbox-5xx
+    metrics_path: /probe
+    params:
+      module: [http_5xx]
+    static_configs:
+      - targets:
+          - https://s3.g4st3r.ru
+          - https://minio.g4st3r.ru
+          - https://cloud.g4st3r.ru
+          - https://grafana.g4st3r.ru
+          - https://prometheus.g4st3r.ru
+          - https://alert.g4st3r.ru
+          - https://photos.g4st3r.ru
+          - https://home.g4st3r.ru
+          - https://gitlab.g4st3r.ru
+          - https://openclaw.g4st3r.ru
+          - https://git.g4st3r.ru
+          - https://bw.g4st3r.ru
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115

monitoring/rules/base-alerts.yml

@@ -10,6 +10,15 @@ groups:
           summary: "{{ $labels.instance }} down"
           description: "{{ $labels.job }} on {{ $labels.instance }} is not responding"
 
+      - alert: NodeExporterDown
+        expr: up{job=~"node|revproxy-node"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Node exporter down on {{ $labels.instance }}"
+          description: "node_exporter is not responding"
+
       - alert: HighCPU
         expr: (1 - avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 0.90
         for: 10m
@@ -28,6 +37,24 @@ groups:
           summary: "High memory on {{ $labels.instance }}"
           description: "Memory usage > 90% for 10m"
 
+      - alert: MemAvailableLow
+        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.08
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Low available memory on {{ $labels.instance }}"
+          description: "MemAvailable < 8%"
+
+      - alert: SwapUsageHigh
+        expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) > 0.50
+        for: 20m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Swap usage high on {{ $labels.instance }}"
+          description: "Swap usage > 50%"
+
       - alert: DiskFull
         expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
         for: 15m
@@ -36,3 +63,48 @@ groups:
         annotations:
           summary: "Disk low on {{ $labels.instance }}"
           description: "<10% free on {{ $labels.mountpoint }}"
+
+      - alert: DiskUsageHigh
+        expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"})) > 0.85
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Disk usage >85% on {{ $labels.instance }}"
+          description: "{{ $labels.mountpoint }} usage above 85%"
+
+      - alert: DiskUsageCritical
+        expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay",mountpoint!~"/boot/efi"})) > 0.95
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Disk usage >95% on {{ $labels.instance }}"
+          description: "{{ $labels.mountpoint }} usage above 95%"
+
+      - alert: DockerContainerDown
+        expr: time() - container_last_seen > 120
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container down"
+          description: "A container has not been seen for >2m"
+
+      - alert: HttpEndpointDown
+        expr: probe_success{job="blackbox-http"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "HTTP endpoint down: {{ $labels.instance }}"
+          description: "Blackbox probe failed"
+
+      - alert: HttpEndpoint5xx
+        expr: probe_success{job="blackbox-5xx"} == 1
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "HTTP 5xx detected: {{ $labels.instance }}"
+          description: "Endpoint returned 5xx"