Embed hosts in service assignment Prometheus map

Use the 'target_host' label for host-level service assignments.

roles/base/templates/vhostmap.prom.j2

@@ -21,6 +21,8 @@ Map tying together float services (float_service=) and systemd services
 #}
 {% for service_name, service in services.items() %}
 {% for systemd_service in service.get('systemd_services', []) %}
-svcmap{float_service="{{ service_name }}",service="{{ systemd_service }}"} 1
+{% for h in groups[service_name] %}
+svcmap{float_service="{{ service_name }}",service="{{ systemd_service }}",target_host="{{ h }}"} 1
+{% endfor %}
 {% endfor %}
 {% endfor %}

roles/prometheus/files/rules/rules_services.conf.yml

@@ -13,14 +13,19 @@ groups:
       - record: node_systemd_unit_ok
         expr: sum(node_systemd_unit_state{state="active"}) by (host,name)
 
+      # The global svcmap is identical on all hosts, pick one.
+      # Rewrite the 'target_host' label to just 'host'.
+      - record: global:svcmap
+        expr: label_replace(max(svcmap) without (host,instance),"host","$1","target_host","(.*)")
+
       # By joining the node_systemd_unit_state with svcmap we can find
       # float_services that have at least one failing systemd unit.
       # The result metric has both float_service and service labels.
       - record: service:presence_by_host
-        expr: label_replace(node_systemd_unit_presence,"service","$1","name","(.*)") * on (service,host) group_right svcmap
+        expr: label_replace(node_systemd_unit_presence,"service","$1","name","(.*)") * on (service,host) group_right global:svcmap
 
       - record: service:ok_by_host
-        expr: label_replace(node_systemd_unit_ok,"service","$1","name","(.*)") * on(service,host) group_right svcmap
+        expr: label_replace(node_systemd_unit_ok,"service","$1","name","(.*)") * on(service,host) group_right global:svcmap
 
       # Now we can group the above at the float_service level, and find
       # hosts that have at least one service failing in the group.