Add support for Ubuntu Focal to our mailman ansible

This switch testing of lists.openstack.org to Focal and we make a CGI
env var update to accomodate newer mailman.

Specifically newer mailman's CGI scripts filter env vars that it will
pass through. We were setting MAILMAN_SITE_DIR to vhost our mailman
installs with apache2, but that doesn't pass the filter and is removed.
HOST is passed through so we update our scripts, apache vhost configs,
exim, and init scripts to use the HOST env var instead.

Change-Id: I5c8c70c219669e37b7b75a61001a2b7f7bb0bb6c

inventory/service/host_vars/lists.openstack.org.yaml

@@ -84,7 +84,7 @@ exim_transports:
                                      {post}}' \
                               $local_part
       current_directory = /var/lib/mailman
-      environment = MAILMAN_SITE_DIR=${lookup{${lc:$domain}}lsearch{/etc/mailman/sites}}
+      environment = HOST=${lc:$domain}
       group = list
       home_directory = /var/lib/mailman
       user = list

playbooks/roles/mailman-list/tasks/main.yaml

@@ -4,21 +4,17 @@
 
 - name: Block for multihost lists
   block:
-    - name: Set mailman site root fact
-      set_fact:
-        mm_site_root: "/srv/mailman/{{ mm_site_name }}"
-        cacheable: no
     - name: Check if the site list exists
       command: list_lists --bare
       environment:
-        MAILMAN_SITE_DIR: "{{ mm_site_root }}"
+        HOST: "{{ mm_site_listdomain }}"
       register: listlists
     - name: Create the site list if it doesn't exist
       command:
         cmd: "newlist {% if mailman_test_mode is defined and mailman_test_mode %} -q {% endif %} {{ mm_list_name }} {{ mm_list_admin }} {{ mm_list_password }}"
         stdin: ''
       environment:
-        MAILMAN_SITE_DIR: "{{ mm_site_root }}"
+        HOST: "{{ mm_site_listdomain }}"
       when:
         - ( listlists.stdout_lines | intersect([ mm_list_name ]) | length | int ) == 0
   when: mm_site_name != "_default"

playbooks/roles/mailman-site/tasks/main.yaml

@@ -102,6 +102,7 @@
     name: mailman-list
   vars:
     mm_site_name: "{{ mailman_site.name }}"
+    mm_site_listdomain: "{{ mailman_site.listdomain }}"
     mm_list_name: "{{ site_list.name }}"
     mm_list_description: "{{ site_list.description }}"
     mm_list_admin: "{{ site_list.admin }}"

playbooks/roles/mailman-site/templates/mailman.init.j2

@@ -21,8 +21,8 @@
 
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 DAEMON=/usr/lib/mailman/bin/mailmanctl
-export MAILMAN_SITE_DIR=/srv/mailman/{{ mailman_site.name }}
-PIDFILE=$MAILMAN_SITE_DIR/run/mailman.pid
+export HOST={{ mailman_site.listdomain }}
+PIDFILE=/srv/mailman/{{ mailman_site.name}}/run/mailman.pid
 
 
 test -x $DAEMON || exit 0

playbooks/roles/mailman-site/templates/mailman_multihost.vhost.j2

@@ -34,7 +34,7 @@ Alias /images/mailman/ /usr/share/images/mailman/
     AllowOverride None
     Options ExecCGI
     AddHandler cgi-script .cgi
-    SetEnv MAILMAN_SITE_DIR /srv/mailman/{{ mailman_site.name }}
+    SetEnv HOST {{ mailman_site.listdomain }}
     Order allow,deny
     Allow from all
     <IfVersion >= 2.4>

playbooks/roles/mailman/files/mm_cfg_multihost.py

@@ -1,15 +1,30 @@
 import os
 import sys
 
-if 'MAILMAN_SITE_DIR' not in os.environ:
-  print("Please set MAILMAN_SITE_DIR")
+if 'MAILMAN_SITE_DIR' not in os.environ and 'HOST' not in os.environ:
+  print("Please set MAILMAN_SITE_DIR or HOST")
   # Exit 0 to avoid confusing the dpkg scripts
   sys.exit(0)
 
-sys.path.insert(0, os.path.join(os.environ['MAILMAN_SITE_DIR'], 'etc'))
+site_dir = None
+if 'MAILMAN_SITE_DIR' in os.environ:
+  site_dir = os.environ['MAILMAN_SITE_DIR']
+elif 'HOST' in os.environ:
+  host = os.environ['HOST']
+  with open('/etc/mailman/sites') as f:
+    for line in f:
+      if line.startswith(host + ':'):
+        site_dir = line.split(':')[1].strip()
+
+if not site_dir:
+  print("Site dir not found")
+  # Exit 0 to avoid confusing the dpkg scripts
+  sys.exit(0)
+
+sys.path.insert(0, os.path.join(site_dir, 'etc'))
 from mm_cfg_local import *
 
-VAR_PREFIX      = os.environ['MAILMAN_SITE_DIR']
+VAR_PREFIX      = site_dir
 
 # Useful directories
 LIST_DATA_DIR   = os.path.join(VAR_PREFIX, 'lists')

testinfra/test_lists_o_o.py

@@ -13,17 +13,17 @@
 testinfra_hosts = ['lists.openstack.org']
 
 def test_mm_list_is_present(host):
-    cmd = host.run('MAILMAN_SITE_DIR=/srv/mailman/airship list_lists --bare')
+    cmd = host.run('HOST=lists.airshipit.org list_lists --bare')
     assert 'airship-discuss' in cmd.stdout
 
-    cmd = host.run('MAILMAN_SITE_DIR=/srv/mailman/opendev list_lists --bare')
+    cmd = host.run('HOST=lists.opendev.org list_lists --bare')
     assert 'service-discuss' in cmd.stdout
 
-    cmd = host.run('MAILMAN_SITE_DIR=/srv/mailman/openstack list_lists --bare')
+    cmd = host.run('HOST=lists.openstack.org list_lists --bare')
     assert 'openstack-discuss' in cmd.stdout
 
-    cmd = host.run('MAILMAN_SITE_DIR=/srv/mailman/starlingx list_lists --bare')
+    cmd = host.run('HOST=lists.starlingx.io list_lists --bare')
     assert 'starlingx-discuss' in cmd.stdout
 
-    cmd = host.run('MAILMAN_SITE_DIR=/srv/mailman/zuul list_lists --bare')
+    cmd = host.run('HOST=lists.zuul-ci.org list_lists --bare')
     assert 'zuul-discuss' in cmd.stdout

zuul.d/system-config-run.yaml

@@ -212,7 +212,7 @@
         - name: bridge.openstack.org
           label: ubuntu-bionic
         - name: lists.openstack.org
-          label: ubuntu-xenial
+          label: ubuntu-focal
         - name: lists.katacontainers.io
           label: ubuntu-focal
     required-projects: