diff --git a/docs/reference.md b/docs/reference.md index 74330c43673fec40295e2c2380592f9442096a46..a3c7d4eef1856db5256e6745431e50a736ba798a 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -2710,7 +2710,7 @@ assignments: * For each service, it will define a host group named after the service, whose members are the hosts assigned to the service; * for each network overlay defined in the inventory, it will define a - host group named `overlay-<name>` whose members are the hosts + host group named `overlay_<name>` whose members are the hosts included in that overlay. These groups can then be used to assign service-specific roles to the diff --git a/playbooks/all.yml b/playbooks/all.yml index 941864d6675880d3a88a62ab4b49cedd3ee8ec98..27bc4c56bd58841bd014ee8d0610831cac786d01 100644 --- a/playbooks/all.yml +++ b/playbooks/all.yml @@ -12,7 +12,7 @@ - float-base-auth-server - float-util-vagrant-compat -- hosts: net-overlay +- hosts: net_overlay roles: - float-base-net-overlay @@ -25,21 +25,21 @@ - import_playbook: prometheus.yml -- hosts: log-collector +- hosts: log_collector gather_facts: no roles: - float-infra-log-collector -- hosts: backup-metadata +- hosts: backup_metadata gather_facts: no roles: - float-base-backup-metadata -- hosts: sso-server +- hosts: sso_server roles: - float-infra-sso-server -- hosts: user-meta-server +- hosts: user_meta_server roles: - float-infra-sso-server diff --git a/playbooks/frontend.yml b/playbooks/frontend.yml index e02744fa7ff9b7cc9f2e42ca7c16a463bf138dfd..6c59341d3b9ec0ce7537b3c74209365965860897 100644 --- a/playbooks/frontend.yml +++ b/playbooks/frontend.yml @@ -9,7 +9,7 @@ - float-infra-dns - float-infra-haproxy -- hosts: admin-dashboard +- hosts: admin_dashboard gather_facts: no roles: - float-infra-admin-dashboard @@ -19,7 +19,7 @@ roles: - float-infra-acme -- hosts: reports-collector +- hosts: reports_collector gather_facts: no roles: - float-infra-reports-collector diff --git a/playbooks/prometheus-lts.yml b/playbooks/prometheus-lts.yml index ec29cc28086c702bb6ce0ba83c5ae370306110ff..fa7989fa2f1993abbe855132a64b5e11b864995f 100644 --- a/playbooks/prometheus-lts.yml +++ b/playbooks/prometheus-lts.yml @@ -1,4 +1,4 @@ -- hosts: prometheus-lts +- hosts: prometheus_lts gather_facts: no roles: - float-infra-prometheus-lts diff --git a/plugins/inventory/float.py b/plugins/inventory/float.py index 1a7c2f22120c03ac109e1286cda562330bf5ca6e..4bc2dd27f880dd0f74bde4e66de9a1d8f5953089 100644 --- a/plugins/inventory/float.py +++ b/plugins/inventory/float.py @@ -45,6 +45,12 @@ class ConfigError(Exception): pass +# Convert a string into Python-compatible identifier (suitable for +# Ansible group names). +def _make_identifier(s): + return s.replace('-', '_') + + # Returns the absolute path to a file. If the given path is relative, # it will be evaluated based on the given path_reference. def _abspath(path, relative_to='/'): @@ -156,10 +162,10 @@ def _host_groups(name, inventory, assignments=None): groups = ['all'] groups.extend(inventory['hosts'][name].get('groups', [])) groups.extend( - ('overlay-' + n) for n in _host_net_overlays(name, inventory)) + ('overlay_' + n) for n in _host_net_overlays(name, inventory)) if assignments: groups.extend( - s for s in assignments.get_by_host(name)) + _make_identifier(s) for s in assignments.get_by_host(name)) return groups @@ -543,6 +549,7 @@ def run_scheduler(config): # schedule-related information before feeding them to Ansible. for service_name, service in services.items(): service['name'] = service_name + service['group_name'] = _make_identifier(service_name) service['user'] = f'docker-{service_name}' service['hosts'] = assignments.get_by_service(service_name) if service.get('master_election'): @@ -582,7 +589,7 @@ def run_scheduler(config): # Create the group->hosts map that Ansible needs. Create a dynamic # net-overlay group with hosts that use network overlays. groups = _build_group_map(inventory, assignments) - groups['net-overlay'] = [ + groups['net_overlay'] = [ h for h in inventory['hosts'] if _host_net_overlays(h, inventory)] diff --git a/roles/float-base-net-overlay/tasks/configure_netoverlay_tinc.yml b/roles/float-base-net-overlay/tasks/configure_netoverlay_tinc.yml index 877b9351be00716ef9eb53ad2af217fea31a52b2..df0e0fc0e03ce819d51a33dc061895cd2878da2e 100644 --- a/roles/float-base-net-overlay/tasks/configure_netoverlay_tinc.yml +++ b/roles/float-base-net-overlay/tasks/configure_netoverlay_tinc.yml @@ -29,7 +29,7 @@ copy: dest: "{{ tinc_dir}}/hosts/{{ h | regex_replace('-', '_') }}" content: "{{ hostvars[h]['tinc_host_config'] }}" - loop: "{{ groups['overlay-' + tinc_net] | list }}" + loop: "{{ groups['overlay_' + tinc_net] | list }}" loop_control: loop_var: h when: "'tinc_host_config' in hostvars[h]" diff --git a/roles/float-base-net-overlay/templates/firewall/11net-overlay-raw.j2 b/roles/float-base-net-overlay/templates/firewall/11net-overlay-raw.j2 index b101d42e2f1ede73f584195c8c10ba66ecec175e..8d8d8bfce34c565a4cc6ac9517de4e448f9374f1 100644 --- a/roles/float-base-net-overlay/templates/firewall/11net-overlay-raw.j2 +++ b/roles/float-base-net-overlay/templates/firewall/11net-overlay-raw.j2 @@ -1,6 +1,6 @@ # Allow peer nodes to communicate with our tinc daemon. create_chain allow-vpn-{{ tinc_net }} -{% for h in groups['overlay-' + tinc_net]|sort %} +{% for h in groups['overlay_' + tinc_net]|sort %} {% if h != inventory_hostname %} add_rule4 -A allow-vpn-{{ tinc_net }} -s {{ hostvars[h]['ip'] }} -j CT --notrack {% if hostvars[h].get('ip6') %} diff --git a/roles/float-base-net-overlay/templates/tinc/tinc.conf.j2 b/roles/float-base-net-overlay/templates/tinc/tinc.conf.j2 index 9638f934851dc6aefa196e2d0cdcd0ed884cde93..196168c9e91438937aa37c830ae9d7d57b30f6fd 100644 --- a/roles/float-base-net-overlay/templates/tinc/tinc.conf.j2 +++ b/roles/float-base-net-overlay/templates/tinc/tinc.conf.j2 @@ -14,7 +14,7 @@ Broadcast = no ReplayWindow = 32 # Connect to all other known nodes (full mesh). -{% for host in groups['overlay-' + tinc_net]|sort %} +{% for host in groups['overlay_' + tinc_net]|sort %} ConnectTo = {{ host | regex_replace('-', '_') }} {% endfor %} diff --git a/roles/float-base/templates/firewall/10float.j2 b/roles/float-base/templates/firewall/10float.j2 index 916ccbc074eb01b783af923d6ab846a228be9443..a30ecf8251834375babad5721cf27b45232c092b 100644 --- a/roles/float-base/templates/firewall/10float.j2 +++ b/roles/float-base/templates/firewall/10float.j2 @@ -21,7 +21,7 @@ create_chain {{ chain }} {{ create_chain_from_host_group('allow-cluster', 'all') }} # Chain to allow traffic from hosts running monitoring probers. -{{ create_chain_from_host_group('allow-monitoring', 'prometheus') }} +{{ create_chain_from_host_group('allow-monitoring', services['prometheus'].group_name) }} # Allow traffic from monitoring probers to local services (on the # public IP). diff --git a/roles/float-base/templates/vhostmap.prom.j2 b/roles/float-base/templates/vhostmap.prom.j2 index 8dd0ea217833abf10380f4f14c5565419ffcaee3..385cf7bd14b1a445381d3c9baafc4622114067dc 100644 --- a/roles/float-base/templates/vhostmap.prom.j2 +++ b/roles/float-base/templates/vhostmap.prom.j2 @@ -21,7 +21,7 @@ Map tying together float services (float_service=) and systemd services #} {% for service_name, service in services | dictsort %} {% for systemd_service in service.get('systemd_services', []) %} -{% for h in groups[service_name] %} +{% for h in services[service_name].hosts | sort %} svcmap{float_service="{{ service_name }}",service="{{ systemd_service }}",target_host="{{ h }}"} 1 {% endfor %} {% endfor %} diff --git a/roles/float-infra-acme/templates/certs.yml.j2 b/roles/float-infra-acme/templates/certs.yml.j2 index f270af4efba31f708a83fab38f9715fdf5ef03a2..7710189468d8f74100aabfabb3b34096087316c5 100644 --- a/roles/float-infra-acme/templates/certs.yml.j2 +++ b/roles/float-infra-acme/templates/certs.yml.j2 @@ -11,7 +11,7 @@ {% for d in domain_public %} - "{{ pe.name }}.{{ d }}" {% if pe.get('sharded', False) %} -{% for h in groups[service_name]|sort if hostvars[h].get('shard_id') %} +{% for h in services[service_name].hosts|sort if hostvars[h].get('shard_id') %} - "{{ hostvars[h].shard_id }}.{{ pe.name }}.{{ d }}" {% endfor %} {% endif %} diff --git a/roles/float-infra-dns/templates/dns/infra.yml b/roles/float-infra-dns/templates/dns/infra.yml index 3bc6daa2422a5dc063f81bb1f208d2b116b319cd..a4738b506cf8450d5b4481082b3e641b234b23a3 100644 --- a/roles/float-infra-dns/templates/dns/infra.yml +++ b/roles/float-infra-dns/templates/dns/infra.yml @@ -48,7 +48,7 @@ {% for pe in s.get('public_endpoints', []) if pe.get('name') and not pe.get('skip_dns', False) %} {{ pe.name }}: CNAME www.l.{{ d }}. {% if pe.get('sharded') %} -{% for h in groups[service_name]|sort %} +{% for h in services[service_name].hosts|sort %} {{ hostvars[h]['shard_id'] }}.{{ pe.name }}: CNAME www.l.{{ d }}. {% endfor %} {% endif %} diff --git a/roles/float-infra-haproxy/templates/haproxy.cfg.j2 b/roles/float-infra-haproxy/templates/haproxy.cfg.j2 index 61db7a64de81818b538b73954c5fbda02120f5df..1e88cb2a7910c870fb3af261c8d13fe43731b1a8 100644 --- a/roles/float-infra-haproxy/templates/haproxy.cfg.j2 +++ b/roles/float-infra-haproxy/templates/haproxy.cfg.j2 @@ -33,7 +33,7 @@ backend be_{{ service_name }}_{{ ep.name }}_{{ port }} log global balance leastconn option independant-streams -{% for s in groups[service_name]|sort %} +{% for s in services[service_name].hosts|sort %} server task{{ loop.index -1 }} {{ s }}.{{ service_name }}.{{ domain }}:{{ port }} check fall 3 id {{ loop.index + 999 }} inter 5000 rise 3 slowstart 60000 weight 50{% if ep.get('use_proxy_protocol') %} send-proxy-v2{% endif %} {% endfor %} @@ -49,7 +49,7 @@ backend be_{{ service_name }}_{{ ep.name }}_{{ ep.port }} log global balance leastconn option independant-streams -{% for s in groups[service_name]|sort %} +{% for s in services[service_name].hosts|sort %} server task{{ loop.index -1 }} {{ s }}.{{ service_name }}.{{ domain }}:{{ ep.port }} check fall 3 id {{ loop.index + 999 }} inter 5000 rise 3 slowstart 60000 weight 50{% if ep.get('use_proxy_protocol') %} send-proxy-v2{% endif %} {% endfor %} diff --git a/roles/float-infra-log-collector/templates/elasticsearch/elasticsearch.yml b/roles/float-infra-log-collector/templates/elasticsearch/elasticsearch.yml index 9f6b005186424c38dad0af6e7f49c4e4103ffc20..d2c5649d1851f800a2293a2ae4b7746214f7578f 100644 --- a/roles/float-infra-log-collector/templates/elasticsearch/elasticsearch.yml +++ b/roles/float-infra-log-collector/templates/elasticsearch/elasticsearch.yml @@ -95,6 +95,6 @@ gateway.expected_nodes: 1 # cluster.initial_master_nodes: -{% for s in groups['log-collector']|sort %} +{% for s in services['log-collector'].hosts|sort %} - "{{ s }}" {% endfor %} diff --git a/roles/float-infra-nginx/templates/nginx-upstream.j2 b/roles/float-infra-nginx/templates/nginx-upstream.j2 index b9be8a5e2c51c9b3a6dafa1cc93a3bb371b426fa..caae9648ef01fe55951e33799489e5fa7affd7f7 100644 --- a/roles/float-infra-nginx/templates/nginx-upstream.j2 +++ b/roles/float-infra-nginx/templates/nginx-upstream.j2 @@ -25,7 +25,7 @@ upstream {{ upstream.name }}{% if shard %}_{{ shard }}{% endif %} { {% for upstream in float_http_upstreams.values() | sort(attribute='service_name') %} {% if upstream.sharded %} -{% for h in groups[upstream.service_name]|sort %} +{% for h in services[upstream.service_name].hosts|sort %} {{ config_upstream(upstream, hostvars[h]['shard_id']) }} {% endfor %} {% else %} diff --git a/roles/float-infra-nginx/templates/nginx-vhost.j2 b/roles/float-infra-nginx/templates/nginx-vhost.j2 index 9f6ea566be4f71f6c5ddffbab848d4b2a65145cc..9824e0d3010378c9aa5c8700cfbf7422e315d4a4 100644 --- a/roles/float-infra-nginx/templates/nginx-vhost.j2 +++ b/roles/float-infra-nginx/templates/nginx-vhost.j2 @@ -65,7 +65,7 @@ server { disable generation of the entire virtual host. #} {% set root_upstream = float_http_upstreams[endpoint.float_path_map['/'].float_upstream_name] %} -{% for h in groups[root_upstream.service_name]|sort %} +{% for h in services[root_upstream.service_name].hosts|sort %} {{ config_vhost(endpoint, hostvars[h]['shard_id']) }} {% endfor %} {% else %} diff --git a/roles/float-infra-prometheus-lts/templates/prometheus.yml.j2 b/roles/float-infra-prometheus-lts/templates/prometheus.yml.j2 index 0e699414bdb74fc53b9b0e582e9b9ee80be799a7..ce205938f55580dfc7601b82fb51d8efe55918d7 100644 --- a/roles/float-infra-prometheus-lts/templates/prometheus.yml.j2 +++ b/roles/float-infra-prometheus-lts/templates/prometheus.yml.j2 @@ -1,35 +1,11 @@ -{# Generate static targets for hosts in an Ansible group #} -{% macro targets_for_group(group, port) %} - - targets: -{% for host in groups[group]|sort %} - - "{{ host }}:{{ port }}" -{% endfor %} -{% endmacro %} - {# Generate static targets for hosts in a float service #} -{% macro targets_for_service(group, service_name, port) %} +{% macro targets_for_service(service_name, port) %} - targets: -{% for host in groups[group]|sort %} +{% for host in services[service_name].hosts|sort %} - "{{ host }}.{{ service_name }}.{{ domain }}:{{ port }}" {% endfor %} {% endmacro %} -{# Generate a static_configs entry for a scrape config #} -{% macro static_configs_for_group(group, port, service_name='') %} - static_configs: -{% if service_name %} -{{ targets_for_service(group, service_name, port) }} - labels: - service: "{{ service_name }}" -{% else %} -{{ targets_for_group(group, port) }} -{% endif %} - relabel_configs: - - source_labels: [__address__] - target_label: host - regex: "([^.]*).*:[0-9]+" - replacement: "${1}" -{% endmacro %} global: scrape_interval: "{{ prometheus_lts_scrape_interval }}" @@ -46,7 +22,7 @@ alerting: action: labeldrop alertmanagers: - static_configs: -{{ targets_for_service('prometheus', 'prometheus', 9093) }} +{{ targets_for_service('prometheus', 9093) }} scrape_configs: - job_name: "prometheus-federation" diff --git a/roles/float-infra-prometheus/templates/prometheus.yml.j2 b/roles/float-infra-prometheus/templates/prometheus.yml.j2 index 27a31d0d392c46128696174a446bb94a42edc95d..19fed9559e77846af2e2d7d83eb61cc64752560d 100644 --- a/roles/float-infra-prometheus/templates/prometheus.yml.j2 +++ b/roles/float-infra-prometheus/templates/prometheus.yml.j2 @@ -50,7 +50,7 @@ cert_file: /etc/credentials/x509/prometheus/client/cert.pem key_file: /etc/credentials/x509/prometheus/client/private_key.pem {% endif %} -{{ static_configs_for_group(target_config.get('group', service_name), target_config.port, service_name) }} +{{ static_configs_for_group(target_config.get('group', services[service_name].group_name), target_config.port, service_name) }} {% endmacro %} global: @@ -191,7 +191,7 @@ scrape_configs: {# Additional blackbox probers #} {% for p in prometheus_additional_blackbox_probers | default([]) | sort(attribute='name') %} -{% for prober_host in groups[p.service] | sort %} +{% for prober_host in services[p.service].hosts | sort %} - job_name: "prober_{{ p.name }}_{{ loop.index }}" metrics_path: "/probe" params: @@ -219,7 +219,7 @@ scrape_configs: # - caller provides a list of hosts, or a float service # - caller provides a regexp to turn the hosts into targets #} -{% set custom_blackbox_targets = p.targets if p.targets is defined else groups[p.get('target_service', 'frontend')] %} +{% set custom_blackbox_targets = p.targets if p.targets is defined else services[p.get('target_service', 'frontend')].hosts %} {% for target in custom_blackbox_targets %} - "{{ target | regex_replace('^(.*)$', p.target_regex | default('\\1')) }}" {% endfor %} diff --git a/roles/float-infra-thanos-query-lts/templates/prometheus-lts.yml.j2 b/roles/float-infra-thanos-query-lts/templates/prometheus-lts.yml.j2 index 2476e9a3ca738be0f2cbd711aeec4dc6cf89d737..3fa738072040b0d3a027774829bef989d01694f2 100644 --- a/roles/float-infra-thanos-query-lts/templates/prometheus-lts.yml.j2 +++ b/roles/float-infra-thanos-query-lts/templates/prometheus-lts.yml.j2 @@ -1,9 +1,9 @@ {# Generate static targets for hosts in a float service #} -{% macro targets_for_service(group, service_name, port) %} +{% macro targets_for_service(service_name, port) %} - targets: -{% for host in groups[group]|sort %} +{% for host in services[service_name].hosts|sort %} - "{{ host }}.{{ service_name }}.{{ domain }}:{{ port }}" {% endfor %} {% endmacro %} -{{ targets_for_service('prometheus-lts', 'prometheus-lts', 10911) }} +{{ targets_for_service('prometheus-lts', 10911) }}