From 15e0d6c7df85bf227055af2c61cd20ebd9518c2d Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 12 Feb 2025 09:10:08 -0800 Subject: [PATCH] Move haproxy config into /var/lib/haproxy Rsyslog on Noble has apparmor rules that restrict rsyslog socket creation to /var/lib/*/dev/log. Previously we were configuring haproxy hosts to create an rsyslog socket for haproxy at /var/haproxy/dev/log which doesn't match the apparmor rule so gets denied. To address this we move all the host side haproxy config from /var/haproxy to /var/lib/haproxy. This allows rsyslog to create the socket. To avoid needing to update docker images (for haproxy statsd) and to continue to make the haproxy container itself happy we don't adjust paths on the target side of our bind mounts. This means some things still refer to /var/haproxy but they should all be within containers. I don't believe this will be impactful to existing load balancer servers. We should deploy new content to /var/lib/haproxy then automatically restart services (rsyslog and haproxy container) because their configs are updating. One potential problem with this is rsyslog will restart before the containers do and its log path will have moved. If we are concerned about this we can configure rsyslog to continue to attempt to create the old path in addition to the new path (this will fail on Noble). Change-Id: I4582e6b2dda188583f76265ab78bcb00a302e375 --- doc/source/gitea.rst | 12 ++++++------ .../roles/haproxy/files/rsyslog.d/49-haproxy.conf | 2 +- playbooks/roles/haproxy/tasks/main.yaml | 11 +++++++---- .../roles/haproxy/templates/docker-compose.yaml.j2 | 8 ++++---- testinfra/test_gitea_lb.py | 9 ++++++++- testinfra/test_zuul_lb.py | 7 +++++++ zuul.d/system-config-run.yaml | 4 ++-- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/doc/source/gitea.rst b/doc/source/gitea.rst index 87d5635890..6e471d5911 100644 --- a/doc/source/gitea.rst +++ b/doc/source/gitea.rst @@ -44,22 +44,22 @@ run as root. To see the current status of all servers:: - echo "show stat" | socat /var/haproxy/run/stats stdio + echo "show stat" | socat /var/lib/haproxy/run/stats stdio To disable a server (eg, gitea09):: - echo "disable server balance_git_http/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio - echo "disable server balance_git_https/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio + echo "disable server balance_git_http/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio + echo "disable server balance_git_https/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio To re-enable a server:: - echo "enable server balance_git_http/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio - echo "enable server balance_git_https/gitea09.opendev.org" | socat /var/haproxy/run/stats stdio + echo "enable server balance_git_http/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio + echo "enable server balance_git_https/gitea09.opendev.org" | socat /var/lib/haproxy/run/stats stdio To run these commands and others interactively, issue the prompt command to haproxy:: - socat readline /var/haproxy/run/stats + socat readline /var/lib/haproxy/run/stats prompt Deploy a New Backend diff --git a/playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf b/playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf index 402c122984..e2d6dae420 100644 --- a/playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf +++ b/playbooks/roles/haproxy/files/rsyslog.d/49-haproxy.conf @@ -1,6 +1,6 @@ # Create additional haproxy socket to be mapped into container -$AddUnixListenSocket /var/haproxy/dev/log +$AddUnixListenSocket /var/lib/haproxy/dev/log :programname, startswith, "haproxy" { /var/log/haproxy.log diff --git a/playbooks/roles/haproxy/tasks/main.yaml b/playbooks/roles/haproxy/tasks/main.yaml index c3a5d73a7b..064032e0c8 100644 --- a/playbooks/roles/haproxy/tasks/main.yaml +++ b/playbooks/roles/haproxy/tasks/main.yaml @@ -3,10 +3,13 @@ name: socat state: present -- name: Ensure registry volume directories exists +- name: Ensure haproxy volume directories exists + # Note on the host side we create everything under /var/lib/haproxy to + # make rsyslog apparmor rules for /var/lib/haproxy/dev/log happy. + # But within the containers /var/haproxy paths are still used. file: state: directory - path: "/var/haproxy/{{ item }}" + path: "/var/lib/haproxy/{{ item }}" owner: 1000 group: 1000 loop: @@ -43,7 +46,7 @@ - name: Write haproxy config file template: src: '{{ haproxy_config_template }}' - dest: /var/haproxy/etc/haproxy.cfg + dest: /var/lib/haproxy/etc/haproxy.cfg owner: 1000 group: 1000 mode: 0644 @@ -59,7 +62,7 @@ - name: Copy in OpenDev Infra CA (test only) copy: src: /etc/opendev-ca/ca.crt - dest: /var/haproxy/etc/ + dest: /var/lib/haproxy/etc/ when: _opendev_ca_crt.stat.exists - name: Ensure docker compose configuration directory diff --git a/playbooks/roles/haproxy/templates/docker-compose.yaml.j2 b/playbooks/roles/haproxy/templates/docker-compose.yaml.j2 index 3d0462cc4e..d9e08e88c7 100644 --- a/playbooks/roles/haproxy/templates/docker-compose.yaml.j2 +++ b/playbooks/roles/haproxy/templates/docker-compose.yaml.j2 @@ -35,9 +35,9 @@ services: user: "root:root" network_mode: host volumes: - - /var/haproxy/dev/log:/dev/log - - /var/haproxy/etc:/usr/local/etc/haproxy:ro - - /var/haproxy/run:/var/haproxy/run + - /var/lib/haproxy/dev/log:/dev/log + - /var/lib/haproxy/etc:/usr/local/etc/haproxy:ro + - /var/lib/haproxy/run:/var/haproxy/run logging: driver: journald options: @@ -50,7 +50,7 @@ services: network_mode: host user: "1000:1000" volumes: - - /var/haproxy/run:/var/haproxy/run + - /var/lib/haproxy/run:/var/haproxy/run environment: STATSD_HOST: graphite.opendev.org STATSD_PORT: 8125 diff --git a/testinfra/test_gitea_lb.py b/testinfra/test_gitea_lb.py index 02d7c24334..0b8ae73b87 100644 --- a/testinfra/test_gitea_lb.py +++ b/testinfra/test_gitea_lb.py @@ -50,8 +50,15 @@ def test_more_than_haproxy_maxconn_conns(host): assert 'OpenDev: Free Software Needs Free Tools' in cmd.stdout def test_haproxy_stats(host): - cmd = host.run('echo "show servers state" | socat /var/haproxy/run/stats stdio | ' + cmd = host.run('echo "show servers state" | socat /var/lib/haproxy/run/stats stdio | ' 'tail +3 | awk \'{print $2,$4,$6}\'') assert 'balance_git_http gitea99.opendev.org 2' in cmd.stdout assert 'balance_git_https gitea99.opendev.org 2' in cmd.stdout + +def test_haproxy_logging(host): + # rsyslog is configured to add a unix socket at this path + assert host.file('/var/lib/haproxy/dev/log').is_socket + # Haproxy logs to syslog via the above socket which produces + # this logfile + assert host.file('/var/log/haproxy.log').is_file diff --git a/testinfra/test_zuul_lb.py b/testinfra/test_zuul_lb.py index 2543aee2d4..58b25d3044 100644 --- a/testinfra/test_zuul_lb.py +++ b/testinfra/test_zuul_lb.py @@ -32,3 +32,10 @@ def test_haproxy_statsd_running(host): out = json.loads(cmd.stdout) assert out[0]["State"]["Status"] == "running" assert out[0]["RestartCount"] == 0 + +def test_haproxy_logging(host): + # rsyslog is configured to add a unix socket at this path + assert host.file('/var/lib/haproxy/dev/log').is_socket + # Haproxy logs to syslog via the above socket which produces + # this logfile + assert host.file('/var/log/haproxy.log').is_file diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 84708d9474..3edfe4b124 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -688,7 +688,7 @@ '/var/log/apache2': logs gitea-lb02.opendev.org: host_copy_output: - '/var/haproxy/etc': logs + '/var/lib/haproxy/etc': logs '/var/log/haproxy.log': logs files: - playbooks/bootstrap-bridge.yaml @@ -1020,7 +1020,7 @@ '/etc/hosts': logs zuul-lb02.opendev.org: host_copy_output: - '/var/haproxy/etc': logs + '/var/lib/haproxy/etc': logs '/var/log/haproxy.log': logs files: - playbooks/bootstrap-bridge.yaml