diff --git a/monitoring/blackbox/blackbox.yml b/monitoring/blackbox/blackbox.yml new file mode 100644 index 0000000..56f3fec --- /dev/null +++ b/monitoring/blackbox/blackbox.yml @@ -0,0 +1,19 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [] + method: GET + follow_redirects: true + preferred_ip_protocol: "ip4" + http_5xx: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [500, 501, 502, 503, 504] + method: GET + follow_redirects: true + preferred_ip_protocol: "ip4" diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 2899873..90837a7 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -77,6 +77,14 @@ services: ports: - "9633:9633" + blackbox-exporter: + image: prom/blackbox-exporter:v0.25.0 + restart: unless-stopped + volumes: + - ./blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro + ports: + - "9115:9115" + volumes: prometheus-data: grafana-data: diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index 534d30c..4d1c01c 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -30,3 +30,55 @@ scrape_configs: - job_name: revproxy-node static_configs: - targets: ["10.8.0.1:9100"] + + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - https://s3.g4st3r.ru + - https://minio.g4st3r.ru + - https://cloud.g4st3r.ru + - https://grafana.g4st3r.ru + - https://prometheus.g4st3r.ru + - https://alert.g4st3r.ru + - https://photos.g4st3r.ru + - https://home.g4st3r.ru + - https://gitlab.g4st3r.ru + - https://openclaw.g4st3r.ru + - https://git.g4st3r.ru + - https://bw.g4st3r.ru + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 + + - job_name: blackbox-5xx + metrics_path: /probe + params: + module: [http_5xx] + static_configs: + - targets: + - https://s3.g4st3r.ru + - https://minio.g4st3r.ru + - https://cloud.g4st3r.ru + - https://grafana.g4st3r.ru + - https://prometheus.g4st3r.ru + - https://alert.g4st3r.ru + - https://photos.g4st3r.ru + - https://home.g4st3r.ru + - https://gitlab.g4st3r.ru + - https://openclaw.g4st3r.ru + - https://git.g4st3r.ru + - https://bw.g4st3r.ru + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 diff --git a/monitoring/rules/base-alerts.yml b/monitoring/rules/base-alerts.yml index 2a000d1..75b6efa 100644 --- a/monitoring/rules/base-alerts.yml +++ b/monitoring/rules/base-alerts.yml @@ -90,3 +90,21 @@ groups: annotations: summary: "Container down" description: "A container has not been seen for >2m" + + - alert: HttpEndpointDown + expr: probe_success{job="blackbox-http"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "HTTP endpoint down: {{ $labels.instance }}" + description: "Blackbox probe failed" + + - alert: HttpEndpoint5xx + expr: probe_success{job="blackbox-5xx"} == 1 + for: 1m + labels: + severity: critical + annotations: + summary: "HTTP 5xx detected: {{ $labels.instance }}" + description: "Endpoint returned 5xx"