Add host turndown workflow

Setting turndown: true on the host inventory will prevent services
from being scheduled on that host.

plugins/inventory/float.py

@@ -282,6 +282,16 @@ def _global_dns_map(inventory):
     return dns
 
 
+# Return the hosts that are not available for scheduling, as a
+# Python set.
+def _unavailable_hosts(inventory):
+    unavail = set()
+    for name, values in inventory['hosts'].items():
+        if values.get('turndown'):
+            unavail.add(name)
+    return unavail
+
+
 # Build a group -> hosts map out of an inventory.
 def _build_group_map(inventory, assignments=None):
     group_map = {}
@@ -499,7 +509,8 @@ class Assignments(object):
         return str(self._fwd)
 
     @classmethod
-    def _available_hosts(cls, service, group_map, service_hosts_map):
+    def _available_hosts(cls, service, group_map, service_hosts_map,
+                         unavailable_hosts={}):
         if 'schedule_with' in service:
             return service_hosts_map[service['schedule_with']]
         scheduling_groups = ['all']
@@ -512,7 +523,7 @@ class Assignments(object):
             if g not in group_map:
                 raise Exception(f'The scheduling_group "{g}" is not defined in inventoy')
             available_hosts.update(group_map[g])
-        return list(available_hosts)
+        return list(available_hosts.difference(unavailable_hosts))
 
     @classmethod
     def schedule(cls, services, inventory):
@@ -525,6 +536,7 @@ class Assignments(object):
         """
         service_hosts_map = {}
         service_master_map = {}
+        unavailable_hosts = _unavailable_hosts(inventory)
         group_map = _build_group_map(inventory)
         host_occupation = collections.defaultdict(int)
 
@@ -540,13 +552,16 @@ class Assignments(object):
         for service_name in sorted(services.keys(), key=_sort_key):
             service = services[service_name]
             available_hosts = cls._available_hosts(service, group_map,
-                                                   service_hosts_map)
+                                                   service_hosts_map,
+                                                   unavailable_hosts)
             num_instances = service.get('num_instances', 'all')
             if num_instances == 'all':
                 service_hosts = sorted(available_hosts)
             else:
                 service_hosts = sorted(_binpack(
                     available_hosts, host_occupation, num_instances))
+            if not service_hosts:
+                raise Exception(f'No hosts available to schedule service {service_name}')
             service_hosts_map[service_name] = service_hosts
             for h in service_hosts:
                 host_occupation[h] += 1