Add blackbox exporter and HTTP alerts

monitoring/rules/base-alerts.yml

@@ -90,3 +90,21 @@ groups:
         annotations:
           summary: "Container down"
           description: "A container has not been seen for >2m"
+
+      - alert: HttpEndpointDown
+        expr: probe_success{job="blackbox-http"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "HTTP endpoint down: {{ $labels.instance }}"
+          description: "Blackbox probe failed"
+
+      - alert: HttpEndpoint5xx
+        expr: probe_success{job="blackbox-5xx"} == 1
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "HTTP 5xx detected: {{ $labels.instance }}"
+          description: "Endpoint returned 5xx"