From 608b2fb3d5e5db4acc487d451971b134b0d71afb Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Thu, 3 Feb 2022 23:51:37 +0000
Subject: [PATCH] Add healthcheck probes for prometheus_external_targets
Part of issue #133.
---
.../templates/prometheus.yml.j2 | 35 ++++++++++++++++++-
1 file changed, 34 insertions(+), 1 deletion(-)
diff --git a/roles/float-infra-prometheus/templates/prometheus.yml.j2 b/roles/float-infra-prometheus/templates/prometheus.yml.j2
index 5d86029e..f2e3f670 100644
--- a/roles/float-infra-prometheus/templates/prometheus.yml.j2
+++ b/roles/float-infra-prometheus/templates/prometheus.yml.j2
@@ -128,7 +128,7 @@ scrape_configs:
static_configs:
- targets:
{% for host in service.hosts | sort %}
- - "{{ target_config.get('scheme', 'http') }}://{{ host }}.{{ service_name }}.{{ domain }}:{{ target_config.port }}{{ target_config.metrics_path | default('/metrics') }}"
+ - "{{ target_config.scheme | default('http') }}://{{ host }}.{{ service_name }}.{{ domain }}:{{ target_config.port }}{{ target_config.metrics_path | default('/metrics') }}"
{% endfor %}
labels:
zone: internal
@@ -140,6 +140,39 @@ scrape_configs:
{% endfor %}
{% endfor %}
+{# Health checks for external targets #}
+{% for target_config in prometheus_external_targets|default([])|sort(attribute='name') %}
+ - job_name: "prober_health_{{ target_config.name | replace('-', '_') }}_{{ prober_idx }}_{{ loop.index }}"
+ metrics_path: "/probe"
+ params:
+ module:
+ - http_health_{{ target_config.healthcheck_http_method | default('HEAD') | lower }}
+ relabel_configs:
+ - source_labels: [__address__]
+ target_label: host
+ regex: "https?://([^.:/]*).*"
+ replacement: "${1}"
+ - source_labels: [__address__]
+ target_label: __param_target
+ - source_labels: [__param_target]
+ target_label: instance
+ - target_label: __address__
+ replacement: {{ prober_host }}.prometheus.{{ domain }}:9115
+ - target_label: prober_host
+ replacement: {{ prober_host }}
+ static_configs:
+ - targets:
+{% for target in target_config.targets | sort %}
+ - "{{ target_config.scheme | default('http') }}://{{ target }}{{ target_config.metrics_path | default('/metrics') }}"
+{% endfor %}
+ labels:
+ zone: external
+ probe: health
+ probeset: health
+ prober_float_service: prometheus
+ float_job: "{{ target_config.name }}"
+{% endfor %}
+
- job_name: "prober_ping_{{ loop.index }}"
metrics_path: "/probe"
params:
--
GitLab