prometheus-lts: add Thanos support

* Configure 'thanos sidecar' alongside 'prometheus-lts' to answer
queries via gRPC
* Add the new sidecar to existing sidecars for 'thanos query' on
'prometheus' hosts to find

playbooks/monitor.yml

 - hosts: prometheus-lts
   roles:
     - prometheus-lts
+
+# Let 'thanos query' on 'prometheus' hosts know about prometheus-lts
+- hosts: prometheus
+  roles:
+    - thanos-lts

roles/thanos-lts/tasks/main.yml

+---
+
+# Configure Thanos query to ask Prometheus Long Term Storage
+
+- name: Create /etc/thanos and subdirs
+  file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - "/etc/thanos"
+    - "/etc/thanos/store-sd"
+
+- name: Install Thanos query discovery files
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/thanos/store-sd/{{ item }}"
+  loop:
+    - "prometheus-lts.yml"

roles/thanos-lts/templates/prometheus-lts.yml.j2

+{# Generate static targets for hosts in a float service #}
+{% macro targets_for_service(group, service_name, port) %}
+      - targets:
+{% for host in groups[group]|sort %}
+          - "{{ host }}.{{ service_name }}.{{ domain }}:{{ port }}"
+{% endfor %}
+{% endmacro %}
+
+{{ targets_for_service('prometheus-lts', 'prometheus-lts', 10911) }}

services.monitor.yml

 # Prometheus LTS pulls from one of the available (shorter term) 'prometheus' servers
+# The Thanos sidecar is used to allow 'thanos query' to query and merge results
+# from Prometheus LTS.
 
 prometheus-lts:
   scheduling_group: prometheus-lts
@@ -13,6 +15,16 @@ prometheus-lts:
         - /etc/prometheus-lts: /etc/prometheus-lts
         - /var/lib/prometheus-lts/metrics2: /var/lib/prometheus-lts/metrics2
       args: "--config.file=/etc/prometheus-lts/prometheus.yml --web.listen-address=0.0.0.0:9099 --storage.tsdb.path=/var/lib/prometheus-lts/metrics2 --storage.tsdb.retention.time={{ prometheus_lts_tsdb_retention }} --web.external-url=https://monitor-lts.{{ domain_public[0] }}"
+    - name: thanos
+      image: registry.git.autistici.org/ai3/docker/thanos:master
+      ports:
+        - 10911 # sidecar grpc
+        - 10912 # sidecar http
+      env:
+        QUERY_OFF: "yes" # The main 'thanos query' service is 'prometheus'
+        SIDECAR_GRPC_ADDRESS: "0.0.0.0:10911"
+        SIDECAR_HTTP_ADDRESS: "0.0.0.0:10912"
+        SIDECAR_FLAGS: "--prometheus.url=http://localhost:9099"
   public_endpoints:
     - name: monitor-lts
       port: 9099
@@ -22,6 +34,11 @@ prometheus-lts:
     - job_name: prometheus-lts
       port: 9099
       scheme: http
+    - job_name: thanos-sidecar-lts
+      port: 10912
+      scheme: http
+  ports:
+    - 10911
   volumes:
     - name: metrics-lts
       path: /var/lib/prometheus-lts