Streamline credential generation with single-level Ansible loops

This makes the credentials role significantly faster, Ansible is
not good at optimizing loops that go across an include_tasks
directive.

We pre-generate some simple lists in float.py for service credentials
that are enabled on each host, to simplify the iteration when
generating certificates.

plugins/inventory/float.py

@@ -27,16 +27,14 @@ from ansible.plugins.inventory import BaseFileInventoryPlugin
 from ansible.module_utils._text import to_native
 
 
-# Common credentials that are not attached to a service but will be
+# Common client credentials that are not attached to a service but will be
 # installed on all hosts (i.e. clients for basic infra services).
 DEFAULT_SERVICE_CREDENTIALS = [
     {
         'name': 'log-client',
-        'enable_server': False,
     },
     {
         'name': 'backup-agent',
-        'enable_server': False,
     },
 ]
 
@@ -352,16 +350,27 @@ def _build_horizontal_upstreams_map(services):
 
 # Return autogenerated host-specific variables.
 def _host_vars(name, inventory, services, assignments):
+    # Set enabled/disabled services for each host, and explode the
+    # service credentials for assigned services so that it is easier
+    # to iterate on them in the 'credentials' Ansible role.
     hv = {
         'float_enabled_services': [],
         'float_disabled_services': [],
         'float_enabled_containers': [],
-        'float_service_credentials_params': {},
+        'float_host_service_credentials': [],
+        'float_host_service_credentials_certs': [],
         'float_host_overlay_networks': _host_net_overlays(name, inventory),
         'float_host_dns_map': _host_service_dns_map(
             name, inventory, services, assignments),
     }
 
+    # Add default client credentials that are present on all hosts.
+    for c in DEFAULT_SERVICE_CREDENTIALS:
+        hv['float_host_service_credentials'].append({'credentials': c})
+        hv['float_host_service_credentials_certs'].append({
+            'credentials': c, 'service': 'LOCAL',
+            'mode': 'client', 'x509_params': {}})
+
     # There isn't necessarily a 1:1 mapping between services and
     # systemd units (in some corner cases of hard separation between
     # roles, for instance), so we need to compute the set difference.
@@ -380,6 +389,18 @@ def _host_vars(name, inventory, services, assignments):
                     _denormalize_container(s, c))
             for u in services[s].get('systemd_services', []):
                 enabled_systemd_units.add(u)
+            for c in services[s].get('service_credentials', []):
+                hv['float_host_service_credentials'].append({
+                    'service': s, 'credentials': c})
+                if c.get('enable_server', True):
+                    params = _service_credential_params(name, s, inventory, assignments)
+                    hv['float_host_service_credentials_certs'].append({
+                        'credentials': c, 'service': s,
+                        'mode': 'server', 'x509_params': params})
+                if c.get('enable_client', True):
+                    hv['float_host_service_credentials_certs'].append({
+                        'credentials': c, 'service': s,
+                        'mode': 'client', 'x509_params': {}})
         else:
             hv['float_disabled_services'].append(s)
             for u in services[s].get('systemd_services', []):
@@ -389,13 +410,6 @@ def _host_vars(name, inventory, services, assignments):
             is_master_var = 'float_%s_is_master' % (service_name_var,)
             hv[is_master_var] = assignments.is_master(s, name)
 
-        for c in services[s].get('service_credentials', []):
-            if c.get('enable_server', True):
-                # The key here is service+cred_name, to allow the same
-                # credentials in different services.
-                cred_key = '%s-%s' % (s, c['name'])
-                hv['float_service_credentials_params'][cred_key] = (
-                    _service_credential_params(name, s, inventory, assignments))
 
     hv['float_disabled_systemd_units'] = list(
         disabled_systemd_units.difference(enabled_systemd_units))
@@ -533,7 +547,6 @@ def run_scheduler(config):
     all_vars.update({
         'float_plugin_loaded': True,
         'services': services,
-        'default_service_credentials': DEFAULT_SERVICE_CREDENTIALS,
         'float_global_dns_map': _global_dns_map(inventory),
         # The following variables are just used for debugging purposes (dashboards).
         'float_service_assignments': assignments._fwd,

roles/credentials/tasks/install_cert.yml

----
-
-- set_fact:
-    x509_params: "{{ float_service_credentials_params[service_name_iter + '-' + credentials.name] | default({}) }}"
-  when: "service_name_iter is defined"
-
-- file:
-    path: "/etc/credentials/x509/{{ credentials.name }}/{{ mode }}"
-    state: directory
-
-- name: "Check the certificate for {{ credentials.name }}/{{ mode }}"
-  x509_csr:
-    credentials_name: "{{ credentials.name }}"
-    domain: "{{ domain }}"
-    mode: "{{ mode }}"
-    params: "{{ x509_params|default({}) }}"
-    private_key_path: "/etc/credentials/x509/{{ credentials.name }}/{{ mode }}/private_key.pem"
-    cert_path: "/etc/credentials/x509/{{ credentials.name }}/{{ mode }}/cert.pem"
-    ca_cert_path: "/etc/credentials/x509/{{ credentials.name }}/ca.pem"
-    check: true
-  check_mode: no
-  register: x509_should_update
-
-# TODO: set the right permissions (credentials.name-credentials)
-- name: "Create the CSR for {{ credentials.name }}/{{ mode }}"
-  x509_csr:
-    credentials_name: "{{ credentials.name }}"
-    domain: "{{ domain }}"
-    mode: "{{ mode }}"
-    params: "{{ x509_params|default({}) }}"
-    private_key_path: "/etc/credentials/x509/{{ credentials.name }}/{{ mode }}/private_key.pem"
-    check: false
-  when: "x509_should_update.changed"
-  register: x509_csr
-
-- name: "Create the certificate for {{ credentials.name }}/{{ mode }}"
-  x509_sign:
-    csr: "{{ x509_csr.csr }}"
-    mode: "{{ mode }}"
-    ca_cert_path: "{{ credentials_dir }}/x509/ca.pem"
-    ca_key_path: "{{ credentials_dir }}/x509/ca_private_key.pem"
-  when: "x509_should_update.changed"
-  register: x509_sign
-
-- name: "Install the signed certificate for {{ credentials.name }}/{{ mode }}"
-  copy:
-    dest: "/etc/credentials/x509/{{ credentials.name }}/{{ mode }}/cert.pem"
-    content: "{{ x509_sign.cert }}"
-    mode: 0644
-  when: "x509_sign.changed"
-
-- name: Set permissions on the private key
-  file:
-    path: "/etc/credentials/x509/{{ credentials.name }}/{{ mode }}/private_key.pem"
-    group: "{{ credentials.name }}-credentials"
-    mode: 0640

roles/credentials/tasks/install_certs.yml

+---
+
+#- set_fact:
+#    x509_params: "{{ float_service_credentials_params[service_name_item + '-' + credentials.name] | default({}) }}"
+#  when: "service_name_item is defined"
+
+- file:
+    path: "/etc/credentials/x509/{{ item.credentials.name }}/{{ item.mode }}"
+    state: directory
+  loop: "{{ float_host_service_credentials_certs }}"
+
+- name: "Check the certificate for {{ item.credentials.name }}/{{ item.mode }}"
+  x509_csr:
+    credentials_name: "{{ item.credentials.name }}"
+    domain: "{{ domain }}"
+    mode: "{{ item.mode }}"
+    params: "{{ item.x509_params|default({}) }}"
+    private_key_path: "/etc/credentials/x509/{{ item.credentials.name }}/{{ item.mode }}/private_key.pem"
+    cert_path: "/etc/credentials/x509/{{ item.credentials.name }}/{{ item.mode }}/cert.pem"
+    ca_cert_path: "/etc/credentials/x509/{{ item.credentials.name }}/ca.pem"
+    check: true
+  loop: "{{ float_host_service_credentials_certs }}"
+  check_mode: no
+  register: x509_should_update
+
+# TODO: set the right permissions (credentials.name-credentials)
+- name: "Create the CSR for {{ item.0.credentials.name }}/{{ item.0.mode }}"
+  x509_csr:
+    credentials_name: "{{ item.0.credentials.name }}"
+    domain: "{{ domain }}"
+    mode: "{{ item.0.mode }}"
+    params: "{{ item.0.x509_params|default({}) }}"
+    private_key_path: "/etc/credentials/x509/{{ item.0.credentials.name }}/{{ item.0.mode }}/private_key.pem"
+    check: false
+  when: "item.1.changed"
+  loop: "{{ float_host_service_credentials_certs | zip(x509_should_update.results) | list }}"
+  register: x509_csr
+
+- name: "Create the certificate for {{ item.0.credentials.name }}/{{ item.0.mode }}"
+  x509_sign:
+    csr: "{{ item.1.csr }}"
+    mode: "{{ item.0.mode }}"
+    ca_cert_path: "{{ credentials_dir }}/x509/ca.pem"
+    ca_key_path: "{{ credentials_dir }}/x509/ca_private_key.pem"
+  when: "item.1.changed"
+  loop: "{{ float_host_service_credentials_certs | zip(x509_csr.results) | list }}"
+  register: x509_sign
+
+- name: "Install the signed certificate for {{ item.0.credentials.name }}/{{ item.0.mode }}"
+  copy:
+    dest: "/etc/credentials/x509/{{ item.0.credentials.name }}/{{ item.0.mode }}/cert.pem"
+    content: "{{ item.1.cert }}"
+    mode: 0644
+  when: "item.1.changed"
+  loop: "{{ float_host_service_credentials_certs | zip(x509_sign.results) | list }}"
+
+- name: Set permissions on the private key
+  file:
+    path: "/etc/credentials/x509/{{ item.credentials.name }}/{{ item.mode }}/private_key.pem"
+    group: "{{ item.credentials.name }}-credentials"
+    mode: 0640
+  loop: "{{ float_host_service_credentials_certs }}"
+

roles/credentials/tasks/install_credentials.yml

 ---
 
-- name: "Create {{ credentials.name }}-credentials group"
+- name: "Create service credentials group"
   group:
-    name: "{{ credentials.name }}-credentials"
+    name: "{{ item.credentials.name }}-credentials"
     system: true
+  loop: "{{ float_host_service_credentials }}"
 
-- name: Create service credentials dir
+- name: "Create service credentials dirs"
   file:
-    path: "/etc/credentials/x509/{{ credentials.name }}"
+    path: "/etc/credentials/x509/{{ item.credentials.name }}"
     state: directory
+  loop: "{{ float_host_service_credentials }}"
 
 - name: Copy CA
   copy:
     src: "{{ credentials_dir }}/x509/ca.pem"
-    dest: "/etc/credentials/x509/{{ credentials.name }}/ca.pem"
+    dest: "/etc/credentials/x509/{{ item.credentials.name }}/ca.pem"
     owner: root
     group: root
     mode: 0644
+  loop: "{{ float_host_service_credentials }}"
 
-- include_tasks: install_cert.yml
-  with_items:
-    - client
-    - server
-  when: "credentials.get('enable_' + mode, True)"
-  loop_control:
-    loop_var: mode

roles/credentials/tasks/install_service.yml

----
-
-- include_tasks: install_credentials.yml
-  with_items: "{{ services[service_name_iter].get('service_credentials', []) }}"
-  loop_control:
-    loop_var: credentials

roles/credentials/tasks/main.yml

@@ -33,15 +33,8 @@
     name: x509ca
     state: present
 
-- include_tasks: install_credentials.yml
-  with_items: "{{ default_service_credentials|default([]) }}"
-  loop_control:
-    loop_var: credentials
-
-- include_tasks: install_service.yml
-  with_items: "{{ float_enabled_services }}"
-  loop_control:
-    loop_var: service_name_iter
+- import_tasks: install_credentials.yml
+- import_tasks: install_certs.yml
 
 # Remove credentials that shouldn't be here.
 # - file: path="/etc/credentials/x509/{{ item.1.name }}" state=absent