Add healthcheck probes for prometheus_external_targets

Part of issue #133.

roles/float-infra-prometheus/templates/prometheus.yml.j2

@@ -128,7 +128,7 @@ scrape_configs:
     static_configs:
       - targets:
 {% for host in service.hosts | sort %}
-        - "{{ target_config.get('scheme', 'http') }}://{{ host }}.{{ service_name }}.{{ domain }}:{{ target_config.port }}{{ target_config.metrics_path | default('/metrics') }}"
+        - "{{ target_config.scheme | default('http') }}://{{ host }}.{{ service_name }}.{{ domain }}:{{ target_config.port }}{{ target_config.metrics_path | default('/metrics') }}"
 {% endfor %}
         labels:
           zone: internal
@@ -140,6 +140,39 @@ scrape_configs:
 {% endfor %}
 {% endfor %}
 
+{# Health checks for external targets #}
+{% for target_config in prometheus_external_targets|default([])|sort(attribute='name') %}
+  - job_name: "prober_health_{{ target_config.name | replace('-', '_') }}_{{ prober_idx }}_{{ loop.index }}"
+    metrics_path: "/probe"
+    params:
+      module:
+        - http_health_{{ target_config.healthcheck_http_method | default('HEAD') | lower }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: host
+        regex: "https?://([^.:/]*).*"
+        replacement: "${1}"
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: {{ prober_host }}.prometheus.{{ domain }}:9115
+      - target_label: prober_host
+        replacement: {{ prober_host }}
+    static_configs:
+      - targets:
+{% for target in target_config.targets | sort %}
+        - "{{ target_config.scheme | default('http') }}://{{ target }}{{ target_config.metrics_path | default('/metrics') }}"
+{% endfor %}
+        labels:
+          zone: external
+          probe: health
+          probeset: health
+          prober_float_service: prometheus
+          float_job: "{{ target_config.name }}"
+{% endfor %}
+
   - job_name: "prober_ping_{{ loop.index }}"
     metrics_path: "/probe"
     params: