Merge branch 'newsletter' into 'master'

"noise" newsletter service

See merge request !246

passwords.mail.yml

@@ -23,3 +23,6 @@
 - name: rt4_mysql_password
   description: MySQL password for rt4 (helpdesk)
 
+- name: noise_subscribe_form_secret
+  description: Secret for noise mailman subscription form
+

playbooks/mail.yml

@@ -24,6 +24,11 @@
   roles:
     - mailman
 
+- hosts: noise
+  gather_facts: no
+  roles:
+    - noise
+
 - hosts: helpdesk
   gather_facts: no
   roles:

roles/mailman/tasks/main.yml

@@ -35,20 +35,18 @@
   template:
     src: "{{ item }}.j2"
     dest: "/etc/mailman/{{ item }}"
-  with_items:
+  loop:
     - mm_cfg.py
     - mailman_api.conf
     - domains
   notify: "reload mailman"
 
-
 - name: Set Mailman forced parameters
   copy:
     src: forced_params
     dest: /etc/mailman/forced_params 
     mode: 0644
 
-
 - name: Create placeholder domains file
   file:
     path: /etc/mailman/domains

roles/mailman/templates/mailman_api.conf.j2

@@ -4,15 +4,14 @@ SSL_CERT = '/etc/credentials/x509/mailman/server/cert.pem'
 SSL_KEY = '/etc/credentials/x509/mailman/server/private_key.pem'
 SSL_CA = '/etc/credentials/x509/mailman/ca.pem'
 
-SSO_LOGIN_SERVER = 'https://login.{{ domain_public[0] }}'
 SSO_SERVICE = 'mailman.{{ domain }}/'
 SSO_DOMAIN = '{{ domain_public[0] }}'
 SSO_GROUPS = ['admins', 'users']
 
 TLS_AUTH_ACLS = [
     ('/api/pwreset', r'(accounts|accountadmin)\.{{ domain | regex_escape }}'),
-    ('/api/create_list', r'mailman\.{{ domain | regex_escape }}'),
-    ('/api/get_list_attrs', r'mailman\.{{ domain | regex_escape }}'),
+    ('/api/create_list', r'account-automation\.{{ domain | regex_escape }}'),
+    ('/api/get_list_attrs', r'account-automation\.{{ domain | regex_escape }}'),
 ]
 
 MAILMAN_URL_HOST = 'www.autistici.org'

roles/noise/defaults/main.yml

+---
+
+noise_smtp:
+  host: "mail-frontend.{{ domain }}"
+  port: 25
+

roles/noise/files/forced_params

+send_reminders = 0
+gateway_to_news = 0
+gateway_to_mail = 0
+dmarc_moderation_action = 1
+respond_to_post_requests = 0
+default_member_moderation = 1
+generic_nonmember_action = 3
+member_moderation_action = 2
+advertised = False

roles/noise/handlers/main.yml

+---
+
+- listen: reload mailman-noise
+  systemd:
+    name: docker-noise-mailman
+    state: restarted
+

roles/noise/tasks/main.yml

+---
+
+- name: Create Mailman data directory
+  file:
+    path: /var/lib/mailman-noise
+    state: directory
+    owner: docker-noise
+    group: docker-noise
+    mode: 0700
+
+- name: Create Mailman data subdirectories
+  file:
+    path: "/var/lib/mailman-noise/{{ item }}"
+    state: directory
+    owner: docker-noise
+    group: docker-noise
+    mode: 0700
+  with_items:
+    - archives
+    - archives/public
+    - archives/private
+    - data
+    - lists
+    - locks
+    - logs
+    - qfiles
+    - spam
+
+- name: Create Mailman config directory
+  file:
+    path: /etc/mailman-noise
+    state: directory
+
+- name: Configure Mailman
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/mailman-noise/{{ item }}"
+  loop:
+    - mm_cfg.py
+    - mailman_api.conf
+  notify: "reload mailman-noise"
+
+- name: Set Mailman forced parameters
+  copy:
+    src: forced_params
+    dest: /etc/mailman-noise/forced_params 
+    mode: 0644
+
+- name: Install backup source
+  template:
+    src: "backup/mailman-noise-source.yml.j2"
+    dest: "/etc/tabacco/sources/mailman-noise.yml"
+

roles/noise/templates/backup/mailman-noise-source.yml.j2

+---
+
+- name: mailman-noise
+  handler: file
+  schedule: "@random_every 1d"
+  params:
+    path: /var/lib/mailman-noise

roles/noise/templates/mailman_api.conf.j2

+# Configuration for python-mailman-api.
+
+SSL_CERT = '/etc/credentials/x509/noise/server/cert.pem'
+SSL_KEY = '/etc/credentials/x509/noise/server/private_key.pem'
+SSL_CA = '/etc/credentials/x509/noise/ca.pem'
+
+SSO_SERVICE = 'noise.{{ domain }}/'
+SSO_DOMAIN = '{{ domain_public[0] }}'
+SSO_GROUPS = ['admins', 'users']
+
+TLS_AUTH_ACLS = [
+    ('/api/pwreset', r'(accounts|accountadmin)\.{{ domain | regex_escape }}'),
+    ('/api/create_list', r'account-automation\.{{ domain | regex_escape }}'),
+    ('/api/get_list_attrs', r'account-automation\.{{ domain | regex_escape }}'),
+]
+
+MAILMAN_URL_HOST = 'noise.autistici.org'
+

roles/noise/templates/mm_cfg.py.j2

+# -*- python -*-
+
+# Copyright (C) 1998,1999,2000 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+"""This is the module which takes your site-specific settings.
+
+From a raw distribution it should be copied to mm_cfg.py.  If you
+already have an mm_cfg.py, be careful to add in only the new settings
+you want.  The complete set of distributed defaults, with annotation,
+are in ./Defaults.  In mm_cfg, override only those you want to
+change, after the
+
+  from Defaults import *
+
+line (see below).
+
+Note that these are just default settings - many can be overridden via the
+admin and user interfaces on a per-list or per-user basis.
+
+Note also that some of the settings are resolved against the active list
+setting by using the value as a format string against the
+list-instance-object's dictionary - see the distributed value of
+DEFAULT_MSG_FOOTER for an example."""
+
+
+#######################################################
+#    Here's where we get the distributed defaults.    #
+
+from Defaults import *
+
+##############################################################
+# Put YOUR site-specific configuration below, in mm_cfg.py . #
+# See Defaults.py for explanations of the values.            #
+
+virtualhosts = {{ noise_domains | default(domain_public) | list | tojson }}
+
+# -------------------------------------------------------------
+# The name of the list Mailman uses to send password reminders
+# and similar. Don't change if you want mailman-owner to be
+# a valid local part.
+MAILMAN_SITE_LIST = 'mailman'
+
+# -------------------------------------------------------------
+# If you change these, you have to configure your http server
+# accordingly (Alias and ScriptAlias directives in most httpds)
+IMAGE_LOGOS = '/mailman/images/'
+
+DEFAULT_URL_PATTERN = 'https://%s/mailman/'
+PRIVATE_ARCHIVE_URL = '/mailman/private/%(listname)s'
+# PUBLIC_EXTERNAL_ARCHIVER = '/usr/sbin/sendmail -i -f noreply@autistici.org lurker+%(listname)s@lurker.m.investici.org'
+# PUBLIC_ARCHIVE_URL = 'http://lists.autistici.org/list/%(listname)s.html'
+
+# A/I custom
+SITE_LINK = Yes
+SITE_TEXT = 'A/I Powered'
+SITE_URL = 'https://www.autistici.org/'
+SITE_LOGO = 'static/img/logoR.png'
+
+# -------------------------------------------------------------
+# Default domain for email addresses of newly created MLs
+DEFAULT_EMAIL_HOST = 'noise.autistici.org'
+# -------------------------------------------------------------
+# Default host for web interface of newly created MLs
+DEFAULT_URL_HOST = 'noise.autistici.org'
+# -------------------------------------------------------------
+# Required when setting any of its arguments.
+for h in virtualhosts:
+    add_virtualhost(DEFAULT_URL_HOST, h)
+
+# This is added so that mailman uses it as the host for the 'mailman' list
+# (don't ask!)
+add_virtualhost(DEFAULT_URL_HOST, "noise.{{ domain }}")
+
+# -------------------------------------------------------------
+# The default language for this server.
+DEFAULT_SERVER_LANGUAGE = 'en'
+
+# -------------------------------------------------------------
+# Iirc this was used in pre 2.1, leave it for now
+USE_ENVELOPE_SENDER = 0              # Still used?
+
+# -------------------------------------------------------------
+# Unset send_reminders on newly created lists
+DEFAULT_SEND_REMINDERS = 0
+
+# -------------------------------------------------------------
+# Senza questo si diventa scemi
+# Ma se lo abilitiamo di default permettiamo il reply-storm di
+# utenti confusi... sigh.
+DEFAULT_REPLY_GOES_TO_LIST = 1
+
+# Impedisce l'accumulo eterno di spam in moderazione
+DEFAULT_MAX_DAYS_TO_HOLD = 15
+
+# rimuove le firme Dkim e Dk dalle mail in entrata a mailman
+REMOVE_DKIM_HEADERS = 'YES'
+
+# SMTP through authenticated relay.
+{% if noise_smtp is defined %}
+SMTPHOST = '{{ noise_smtp.host }}'
+SMTPPORT = '{{ noise_smtp.port | default(465) }}'
+{% if noise_smtp.auth_user is defined %}
+SMTP_AUTH = True
+SMTP_USE_TLS = True
+SMTP_USER = '{{ noise_smtp.auth_user }}'
+SMTP_PASSWD = '{{ noise_smtp.auth_password }}'
+{% endif %}
+{% endif %}
+
+# -------------------------------------------------------------
+# Uncomment this if you configured your MTA such that it
+# automatically recognizes newly created lists.
+# (see /usr/share/doc/mailman/README.{EXIM,...})
+# MTA=None   # Misnomer, suppresses alias output on newlist
+
+# -------------------------------------------------------------
+# Uncomment if you use Postfix virtual domains, but be sure to
+# read /usr/share/doc/mailman/README.POSTFIX first.
+MTA='Postfix'
+
+POSTFIX_STYLE_VIRTUAL_DOMAINS = virtualhosts
+
+DEFAULT_ARCHIVE = Off
+DEFAULT_ARCHIVE_PRIVATE = 1
+ARCHIVE_TO_MBOX = 2
+ARCHIVER_OBSCURES_EMAILADDRS = 1
+ARCHIVE_HTML_SANITIZER = 0
+
+# added by ale+blicero to limit recipient per send and avoid spam limit measures
+SMTP_MAX_RCPTS = 50
+
+# Note - if you're looking for something that is imported from mm_cfg, but you
+# didn't find it above, it's probably in /usr/lib/mailman/Mailman/Defaults.py.
+
+DEFAULT_SUBSCRIBE_POLICY = 3
+DEFAULT_LIST_ADVERTISED = No
+
+DMARC_MODERATION_ACTION = 1
+DEFAULT_RESPOND_TO_POST_REQUESTS = 0
+
+DEFAULT_MEMBER_MODERATION = 1
+DEFAULT_GENERIC_NONMEMBER_ACTION = 3
+DEFAULT_FROM_IS_LIST = 1
+DEFAULT_ANONYMOUS_LIST = 1
+
+# Process bounces queue more frequently (default = every 15 min)
+REGISTER_BOUNCES_EVERY = minutes(5)
+
+
+DEFAULT_MAX_MESSAGE_SIZE = 10000
+DEFAULT_FORWARD_AUTO_DISCARDS = No
+DEFAULT_PRIVATE_ROSTER = 2
+DEFAULT_DIGESTABLE = No
+
+DEFAULT_BOUNCE_PROCESSING = Yes
+DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNERS = No
+DEFAULT_BOUNCE_SCORE_THRESHOLD = 2
+DEFAULT_BOUNCE_INFO_STALE_AFTER = 30
+DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS = 2
+DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS_INTERVAL = 2
+DEFAULT_BOUNCE_NOTIFY_OWNER_ON_DISABLE = Yes
+DEFAULT_BOUNCE_NOTIFY_OWNER_ON_REMOVAL = Yes
+
+# Fighting Mailman Subscription Spam
+FORM_LIFETIME = hours(1)
+SUBSCRIBE_FORM_SECRET = "{{ noise_subscribe_form_secret }}"
+SUBSCRIBE_FORM_MIN_TIME = seconds(5)
+
+# Enable LMTP.
+LMTP_HOST = '0.0.0.0'
+LMTP_PORT = 6898
+QRUNNERS.extend([('LMTPRunner', 1),])
+

services.mail.yml

@@ -110,6 +110,40 @@ mail-backend:
       params:
         compress: true
 
+noise:
+  scheduling_group: backend
+  num_instances: 1
+  service_credentials:
+    - name: noise
+  containers:
+    - name: mailman
+      image: registry.git.autistici.org/ai3/docker/mailman:master
+      port: 6897
+      volumes:
+        - /etc/mailman-noise: /config
+        - /var/lib/mailman-noise: /data
+      env:
+        DOMAIN: "{{ domain }}"
+        APACHE_PORT: 6897
+        API_PORT: 6888
+  public_endpoints:
+    - name: noise
+      port: 6897
+      scheme: http
+  monitoring_endpoints:
+    - port: 6997
+      scheme: http
+  ports:
+    - 6888
+    - 6897
+    - 6898
+  volumes:
+    - name: data
+      path: /var/lib/mailman-noise
+      owner: docker-noise
+      group: docker-noise
+      size: 50g
+
 mailman:
   scheduling_group: backend
   service_credentials: