Support multiple HTTP methods for the health check probes

Not all processes respond to HEAD requests on /metrics, so use the new
'healthcheck_http_method' attribute of monitoring_endpoints to select
another method.

roles/float-infra-prometheus/templates/blackbox.yml.j2

 modules:
 
+  # Healthcheck probe to use on internal Prometheus targets. Only use
+  # when the target does not support HTTP methods that do not transfer
+  # data (HEAD / OPTIONS).
+  http_health_get:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: [200]
+      method: GET
+      fail_if_ssl: false
+      fail_if_not_ssl: false
+      preferred_ip_protocol: ip4
+      tls_config:
+        ca_file: /etc/credentials/x509/prometheus/ca.pem
+        cert_file: /etc/credentials/x509/prometheus/client/cert.pem
+        key_file: /etc/credentials/x509/prometheus/client/private_key.pem
+        insecure_skip_verify: false
+
   # Healthcheck probe to use on internal Prometheus targets, makes
   # a simple HEAD request to /metrics to avoid transferring the entire
   # metrics dump (which might be huge).
-  http_health:
+  http_health_head:
     prober: http
     timeout: 5s
     http:
@@ -18,6 +36,25 @@ modules:
         key_file: /etc/credentials/x509/prometheus/client/private_key.pem
         insecure_skip_verify: false
 
+  # Healthcheck probe to use on internal Prometheus targets, makes
+  # a simple OPTIONS request to /metrics to avoid transferring the entire
+  # metrics dump (which might be huge). In particular, Prometheus jobs
+  # themselves do not support the HEAD method, but do understand OPTIONS.
+  http_health_options:
+    prober: http
+    timeout: 5s
+    http:
+      valid_status_codes: [200]
+      method: OPTIONS
+      fail_if_ssl: false
+      fail_if_not_ssl: false
+      preferred_ip_protocol: ip4
+      tls_config:
+        ca_file: /etc/credentials/x509/prometheus/ca.pem
+        cert_file: /etc/credentials/x509/prometheus/client/cert.pem
+        key_file: /etc/credentials/x509/prometheus/client/private_key.pem
+        insecure_skip_verify: false
+
   # Base HTTPS probe that will not verify certificates, used
   # only to check that NGINX is up and running.
   http_base:

roles/float-infra-prometheus/templates/prometheus.yml.j2

@@ -111,7 +111,7 @@ scrape_configs:
     metrics_path: "/probe"
     params:
       module:
-        - http_health
+        - http_health_{{ target_config.healthcheck_http_method | default('HEAD') | lower }}
     relabel_configs:
       - source_labels: [__address__]
         target_label: host

services.prometheus-lts.yml

@@ -34,6 +34,7 @@ prometheus-lts:
   monitoring_endpoints:
     - port: 9099
       scheme: http
+      healtcheck_http_method: OPTIONS
     - port: 10912
       scheme: http
   ports:

services.yml.no-elasticsearch

@@ -143,10 +143,13 @@ prometheus:
   monitoring_endpoints:
     - port: 9090
       scheme: http
+      healtcheck_http_method: OPTIONS
     - port: 9093
       scheme: http
+      healtcheck_http_method: OPTIONS
     - port: 9193
       scheme: http
+      healtcheck_http_method: GET
     - port: 2929
       scheme: https
     - port: 10904