De-noise HostUnreachable signal

roles/float-infra-prometheus/templates/rules/alerts_base.conf.yml

@@ -8,8 +8,8 @@ groups:
   # once the host becomes reachable again, so as to inhibit alerts that
   # might fire immediately after the transition.
   - alert: HostUnreachable
-    expr: min_over_time(host_reachable[10m]) == 0
-    for: 2m
+    expr: min_over_time(smoothed_host_reachable[10m]) == 0
+    for: 1m
     labels:
       severity: warn
     annotations:

roles/float-infra-prometheus/templates/rules/rules_base.conf.yml

@@ -37,5 +37,10 @@ groups:
 
       # Special metric for the ping probe.
       # The 'bool' qualifier makes the greater-than operation not act as a filter.
+      # The 'smoothed' metric verifies that the host has been unreachable for two
+      # consecutive minutes.
       - record: host_reachable
         expr: target:probe_success:ratio{probe="ping"} > bool 0.6
+      - record: smoothed_host_reachable
+        expr: max_over_time(host_reachable[2m])
+