Move haproxy config into /var/lib/haproxy

Rsyslog on Noble has apparmor rules that restrict rsyslog socket
creation to /var/lib/*/dev/log. Previously we were configuring haproxy
hosts to create an rsyslog socket for haproxy at /var/haproxy/dev/log
which doesn't match the apparmor rule so gets denied.

To address this we move all the host side haproxy config from
/var/haproxy to /var/lib/haproxy. This allows rsyslog to create the
socket. To avoid needing to update docker images (for haproxy statsd)
and to continue to make the haproxy container itself happy we don't
adjust paths on the target side of our bind mounts. This means some
things still refer to /var/haproxy but they should all be within
containers.

I don't believe this will be impactful to existing load balancer
servers. We should deploy new content to /var/lib/haproxy then
automatically restart services (rsyslog and haproxy container) because
their configs are updating. One potential problem with this is rsyslog
will restart before the containers do and its log path will have moved.
If we are concerned about this we can configure rsyslog to continue to
attempt to create the old path in addition to the new path (this will
fail on Noble).

Change-Id: I4582e6b2dda188583f76265ab78bcb00a302e375

doc/source/gitea.rst

@@ -44,22 +44,22 @@ run as root.
 
 To see the current status of all servers::
 
-  echo "show stat" | socat /var/haproxy/run/stats stdio
+  echo "show stat" | socat /var/lib/haproxy/run/stats stdio
 
 To disable a server (eg, gitea09)::
 
-  echo "disable server balance_git_http/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio
-  echo "disable server balance_git_https/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio
+  echo "disable server balance_git_http/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio
+  echo "disable server balance_git_https/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio
 
 To re-enable a server::
 
-  echo "enable server balance_git_http/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio
-  echo "enable server balance_git_https/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio
+  echo "enable server balance_git_http/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio
+  echo "enable server balance_git_https/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio
 
 To run these commands and others interactively, issue the prompt
 command to haproxy::
 
-  socat readline /var/haproxy/run/stats
+  socat readline /var/lib/haproxy/run/stats
   prompt
 
 Deploy a New Backend

playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf

@@ -1,6 +1,6 @@
 # Create additional haproxy socket to be mapped into container
 
-$AddUnixListenSocket /var/haproxy/dev/log
+$AddUnixListenSocket /var/lib/haproxy/dev/log
 
 :programname, startswith, "haproxy" {
    /var/log/haproxy.log

playbooks/roles/haproxy/tasks/main.yaml

@@ -3,10 +3,13 @@
     name: socat
     state: present
 
-- name: Ensure registry volume directories exists
+- name: Ensure haproxy volume directories exists
+  # Note on the host side we create everything under /var/lib/haproxy to
+  # make rsyslog apparmor rules for /var/lib/haproxy/dev/log happy.
+  # But within the containers /var/haproxy paths are still used.
   file:
     state: directory
-    path: "/var/haproxy/{{ item }}"
+    path: "/var/lib/haproxy/{{ item }}"
     owner: 1000
     group: 1000
   loop:
@@ -43,7 +46,7 @@
 - name: Write haproxy config file
   template:
     src: '{{ haproxy_config_template }}'
-    dest: /var/haproxy/etc/haproxy.cfg
+    dest: /var/lib/haproxy/etc/haproxy.cfg
     owner: 1000
     group: 1000
     mode: 0644
@@ -59,7 +62,7 @@
 - name: Copy in OpenDev Infra CA (test only)
   copy:
     src: /etc/opendev-ca/ca.crt
-    dest: /var/haproxy/etc/
+    dest: /var/lib/haproxy/etc/
   when: _opendev_ca_crt.stat.exists
 
 - name: Ensure docker compose configuration directory

playbooks/roles/haproxy/templates/docker-compose.yaml.j2

@@ -35,9 +35,9 @@ services:
     user: "root:root"
     network_mode: host
     volumes:
-      - /var/haproxy/dev/log:/dev/log
-      - /var/haproxy/etc:/usr/local/etc/haproxy:ro
-      - /var/haproxy/run:/var/haproxy/run
+      - /var/lib/haproxy/dev/log:/dev/log
+      - /var/lib/haproxy/etc:/usr/local/etc/haproxy:ro
+      - /var/lib/haproxy/run:/var/haproxy/run
     logging:
       driver: journald
       options:
@@ -50,7 +50,7 @@ services:
     network_mode: host
     user: "1000:1000"
     volumes:
-      - /var/haproxy/run:/var/haproxy/run
+      - /var/lib/haproxy/run:/var/haproxy/run
     environment:
       STATSD_HOST: graphite.opendev.org
       STATSD_PORT: 8125

testinfra/test_gitea_lb.py

@@ -50,8 +50,15 @@ def test_more_than_haproxy_maxconn_conns(host):
     assert 'OpenDev: Free Software Needs Free Tools' in cmd.stdout
 
 def test_haproxy_stats(host):
-    cmd = host.run('echo "show servers state" | socat /var/haproxy/run/stats stdio | '
+    cmd = host.run('echo "show servers state" | socat /var/lib/haproxy/run/stats stdio | '
                    'tail +3 | awk \'{print $2,$4,$6}\'')
 
     assert 'balance_git_http gitea99.opendev.org 2' in cmd.stdout
     assert 'balance_git_https gitea99.opendev.org 2' in cmd.stdout
+
+def test_haproxy_logging(host):
+    # rsyslog is configured to add a unix socket at this path
+    assert host.file('/var/lib/haproxy/dev/log').is_socket
+    # Haproxy logs to syslog via the above socket which produces
+    # this logfile
+    assert host.file('/var/log/haproxy.log').is_file

testinfra/test_zuul_lb.py

@@ -32,3 +32,10 @@ def test_haproxy_statsd_running(host):
     out = json.loads(cmd.stdout)
     assert out[0]["State"]["Status"] == "running"
     assert out[0]["RestartCount"] == 0
+
+def test_haproxy_logging(host):
+    # rsyslog is configured to add a unix socket at this path
+    assert host.file('/var/lib/haproxy/dev/log').is_socket
+    # Haproxy logs to syslog via the above socket which produces
+    # this logfile
+    assert host.file('/var/log/haproxy.log').is_file

zuul.d/system-config-run.yaml

@@ -688,7 +688,7 @@
           '/var/log/apache2': logs
       gitea-lb02.opendev.org:
         host_copy_output:
-          '/var/haproxy/etc': logs
+          '/var/lib/haproxy/etc': logs
           '/var/log/haproxy.log': logs
     files:
       - playbooks/bootstrap-bridge.yaml
@@ -1020,7 +1020,7 @@
           '/etc/hosts': logs
       zuul-lb02.opendev.org:
         host_copy_output:
-          '/var/haproxy/etc': logs
+          '/var/lib/haproxy/etc': logs
           '/var/log/haproxy.log': logs
     files:
       - playbooks/bootstrap-bridge.yaml