Allow setting custom scrape_intervals for services and blackbox probes

docs/reference.md

@@ -1575,6 +1575,8 @@ provided:
     specify a regex (with a capture group) to extract back the host
     name from the target; the default regex will extract the short
     host name from URLs and host:port targets.
+* (optionally) a *scrape_interval* if for some reason it should be
+  different than the default *prometheus_probe_scrape_interval*.
 
 So, in the context of the previous example, if we wanted to probe
 another float service called *myservice*, which hypothetically serves
@@ -2032,6 +2034,9 @@ attributes:
 `labels`: An optional dictionary of key/value labels to set for this
 target (they will be added to all metrics scraped from it).
 
+`scrape_interval`: Optionally override the scrape interval for this
+target.
+
 The Prometheus *job* labels for service targets will be automatically
 generated by *float* to include the service name and the endpoint
 port.
@@ -2533,6 +2538,10 @@ instances should scrape their targets (default 10s).
 `prometheus_lts_scrape_interval` sets how often the long-term
 Prometheus instances should scrape the primary ones (default 1m).
 
+`prometheus_probe_scrape_interval` controls the default
+scrape_interval setting for all blackbox probes, and it just defaults
+to the value of prometheus_scrape_interval if unset.
+
 `prometheus_external_targets` allows adding additional targets to Prometheus
 beyond those that are described by the service metadata. It is a list of entries
 with *name*, *targets* attributes. Optionally, you may specify a *scheme*

roles/float-infra-prometheus/templates/prometheus.yml.j2

@@ -44,6 +44,9 @@
 {% macro job_service_config(service_name, target_config) %}
   - job_name: "{{ service_name }}_{{ target_config.port }}"
     scheme: "{{ target_config.get('scheme', 'https') }}"
+{% if target_config.get('scrape_interval') %}
+    scrape_interval: "{{ target_config['scrape_interval'] }}"
+{% endif %}
 {% if target_config.get('metrics_path') %}
     metrics_path: "{{ target_config['metrics_path'] }}"
 {% endif %}
@@ -103,6 +106,7 @@ scrape_configs:
 {# Blackbox probes #}
 {% for prober_host in services['prometheus'].hosts|sort %}
 {% set prober_idx = loop.index %}
+{% set probe_scrape_interval = prometheus_probe_scrape_interval | default(prometheus_scrape_interval) %}
 
 {# Create all the health probes for all the monitored services #}
 {% for service_name, service in services | dictsort %}
@@ -142,6 +146,7 @@ scrape_configs:
 
   - job_name: "prober_ping_{{ loop.index }}"
     metrics_path: "/probe"
+    scrape_interval: "{{ probe_scrape_interval }}"
     params:
       module:
         - ping
@@ -171,6 +176,7 @@ scrape_configs:
 
   - job_name: "prober_https_{{ prober_idx }}"
     metrics_path: "/probe"
+    scrape_interval: "{{ probe_scrape_interval }}"
     params:
       module:
         - http_base
@@ -200,6 +206,7 @@ scrape_configs:
 
   - job_name: "prober_dns_{{ prober_idx }}"
     metrics_path: "/probe"
+    scrape_interval: "{{ probe_scrape_interval }}"
     params:
       module: [dns_toplevel]
     relabel_configs:
@@ -233,6 +240,7 @@ scrape_configs:
 {% for prober_host in services[p.service].hosts | sort %}
   - job_name: "prober_{{ p.name }}_{{ loop.index }}"
     metrics_path: "/probe"
+    scrape_interval: "{{ p.scrape_interval | default(probe_scrape_interval) }}"
     params:
       module:
         - {{ p.module | default(p.name) }}
@@ -280,6 +288,9 @@ scrape_configs:
   - job_name: "{{ target.name }}"
     scheme: "{{ target.scheme | default('http') }}"
     metrics_path: "{{ target.metrics_path | default('/metrics') }}"
+{% if target.scrape_interval is defined %}
+    scrape_interval: "{{ target.scrape_interval }}"
+{% endif %}
     static_configs:
       - targets: {{ target.targets | to_json }}
         labels: