From 962af3e24eb2ac2b10e3ad0d2e8e715d9d1665bc Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 16 Jan 2025 12:03:52 -0800 Subject: [PATCH] Use newer borgbackup on Ubuntu Noble MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python3.12 (which is the python version on Noble) is not compatible with our pinned borgbackup version (1.1.18). We get his errors when building borgbackup on python3.12: ‘PyLongObject’ {aka ‘struct _longobject’} has no member named ‘ob_digit’ We update to 1.2.8 on Noble which is one of the oldest versions claiming python3.12 support. We try to use the oldest version to ensure maximum compatiblity with 1.1.18 on the backup servers. Our CI job should give us decent coverage and then the new paste02 will be the production canary for whether or not these versions are compatible enough with each other. No other servers should be effected in the initial pass. Note there is an upgrade event horizon for using repos with borg<1.2.5 and borg >=1.2.5. It only affects repos that have archives that lack TAMs. My read on it is that newer borg can treat those archives as invalid and unceremoniously delete them. This is a problem if they are valid archives and don't have a TAM. I suspect we will avoid this problem because borg >= 1.0.9 creates TAMs with archives and we prune our archives so older ones should be long gone. More info on this can be found in these documents and reviewers are encouraged to read them: https://borgbackup.readthedocs.io/en/1.2-maint/changes.html#pre-1-2-5-archives-spoofing-vulnerability-cve-2023-36811 https://borgbackup.readthedocs.io/en/1.2-maint/changes.html#borg-1-1-x-to-1-2-x https://borgbackup.readthedocs.io/en/stable/usage/upgrade.html#borg-1-x-y-upgrades I have left a todo for us to upgrade all of the services to 1.2.8 that can run it (it requires python3.8 or newer so Focal or newer) but for now we are taking baby steps. Change-Id: I0c5ca758149b85aeec5321a704300489a57a3cc1 --- .../borg-backup/templates/borg-backup.j2 | 3 ++ .../roles/borg-backup/templates/borg-mount.j2 | 2 ++ .../roles/install-borg/defaults/main.yaml | 1 - playbooks/roles/install-borg/tasks/main.yaml | 31 +++++++++++++++++-- playbooks/zuul/templates/gate-groups.yaml.j2 | 1 + .../zuul/templates/group_vars/all.yaml.j2 | 5 +++ testinfra/test_borg_backups.py | 17 ++++++++-- zuul.d/system-config-run.yaml | 5 +++ 8 files changed, 59 insertions(+), 6 deletions(-) delete mode 100644 playbooks/roles/install-borg/defaults/main.yaml diff --git a/playbooks/roles/borg-backup/templates/borg-backup.j2 b/playbooks/roles/borg-backup/templates/borg-backup.j2 index fa1289a529..9bfe8816e9 100644 --- a/playbooks/roles/borg-backup/templates/borg-backup.j2 +++ b/playbooks/roles/borg-backup/templates/borg-backup.j2 @@ -25,6 +25,9 @@ export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=1 # Backup the most important directories into an archive named after # the machine this script is currently running on: +# TODO(clarkb) Borg 1.2 deprecated exclude paths starting with a leading / +# Borg 1.2 should strip them off for us, but we should clean up our excludes +# after everything is running 1.2 or newer. ${BORG_CREATE} \ {% for item in borg_backup_excludes + borg_backup_excludes_extra -%} --exclude '{{ item }}' \ diff --git a/playbooks/roles/borg-backup/templates/borg-mount.j2 b/playbooks/roles/borg-backup/templates/borg-mount.j2 index 09d086e2d6..a24363a061 100644 --- a/playbooks/roles/borg-backup/templates/borg-mount.j2 +++ b/playbooks/roles/borg-backup/templates/borg-mount.j2 @@ -1,5 +1,7 @@ #!/bin/bash +set -e + if [ -z "$1" ]; then echo "Must specify backup host" exit 1 diff --git a/playbooks/roles/install-borg/defaults/main.yaml b/playbooks/roles/install-borg/defaults/main.yaml deleted file mode 100644 index 9fcf2d0257..0000000000 --- a/playbooks/roles/install-borg/defaults/main.yaml +++ /dev/null @@ -1 +0,0 @@ -borg_version: 1.1.18 diff --git a/playbooks/roles/install-borg/tasks/main.yaml b/playbooks/roles/install-borg/tasks/main.yaml index e80edce9c7..cdcc310fda 100644 --- a/playbooks/roles/install-borg/tasks/main.yaml +++ b/playbooks/roles/install-borg/tasks/main.yaml @@ -13,9 +13,36 @@ - libacl1-dev - libacl1 - build-essential + - pkg-config + +- name: Install Noble specific fuse build deps + package: + name: + - libfuse3-dev + - fuse3 + when: ansible_distribution_release == 'noble' + +- name: Install fuse build deps for everything else + package: + name: - libfuse-dev - fuse - - pkg-config + when: ansible_distribution_release != 'noble' + +# Noble's python3.12 can't run 1.1.18 so we special case a newer +# version here. +# TODO(clarkb) Update borg across the board. +- name: Set fuse and version variables for Noble + set_fact: + _borg_version: "{{ borg_version | default('1.2.8', true) }}" + _fuse_extra: "pyfuse3" + when: ansible_distribution_release == 'noble' + +- name: Set fuse and version variables for everything else + set_fact: + _borg_version: "{{ borg_version | default('1.1.18', true) }}" + _fuse_extra: "fuse" + when: ansible_distribution_release != 'noble' - name: Create venv include_role: @@ -29,5 +56,5 @@ # but the requirements don't bring it in. name: - cython - - 'borgbackup[fuse]=={{ borg_version }}' + - "borgbackup[{{ _fuse_extra }}]=={{ _borg_version }}" virtualenv: /opt/borg diff --git a/playbooks/zuul/templates/gate-groups.yaml.j2 b/playbooks/zuul/templates/gate-groups.yaml.j2 index 77a33da0a9..a9193d7aba 100644 --- a/playbooks/zuul/templates/gate-groups.yaml.j2 +++ b/playbooks/zuul/templates/gate-groups.yaml.j2 @@ -23,6 +23,7 @@ groups: - borg-backup-bionic.opendev.org - borg-backup-focal.opendev.org - borg-backup-jammy.opendev.org + - borg-backup-noble.opendev.org kerberos-kdc: - kdc-primary.opendev.org diff --git a/playbooks/zuul/templates/group_vars/all.yaml.j2 b/playbooks/zuul/templates/group_vars/all.yaml.j2 index cb80bcbd6a..5cc64bea06 100644 --- a/playbooks/zuul/templates/group_vars/all.yaml.j2 +++ b/playbooks/zuul/templates/group_vars/all.yaml.j2 @@ -12,3 +12,8 @@ iptables_test_public_tcp_ports: {{ iptables_test_public_tcp_ports }} iptables_egress_rules: - -o lo -j ACCEPT - -p tcp -m tcp --dport 25 --tcp-flags FIN,SYN,RST,ACK SYN -j REJECT --reject-with tcp-reset +# This is the file we log our backups to. This means the file is being +# changed while backups run which can lead to a warning and non zero exit +# code from borg. Just ignore it as we don't need to backup the file. +borg_backup_excludes_extra: + - /var/log/borg-backup-borg-backup01.region.provider.opendev.org.log diff --git a/testinfra/test_borg_backups.py b/testinfra/test_borg_backups.py index 17b37602a1..12ae65a7ec 100644 --- a/testinfra/test_borg_backups.py +++ b/testinfra/test_borg_backups.py @@ -18,7 +18,8 @@ import pytest testinfra_hosts = ['borg-backup01.region.provider.opendev.org', 'borg-backup-bionic.opendev.org', 'borg-backup-focal.opendev.org', - 'borg-backup-jammy.opendev.org'] + 'borg-backup-jammy.opendev.org', + 'borg-backup-noble.opendev.org'] def test_borg_installed(host): @@ -29,7 +30,11 @@ def test_borg_installed(host): assert cmd.succeeded # NOTE(ianw): deliberately pinned; we want to be careful if we # update that the new version is compatible with old repos. - assert '1.1.18' in cmd.stdout + hostname = host.backend.get_hostname() + if hostname == 'borg-backup-noble.opendev.org': + assert '1.2.8' in cmd.stdout + else: + assert '1.1.18' in cmd.stdout def test_borg_server_users(host): hostname = host.backend.get_hostname() @@ -38,7 +43,8 @@ def test_borg_server_users(host): for username in ('borg-borg-backup-bionic', 'borg-borg-backup-focal', - 'borg-borg-backup-jammy'): + 'borg-borg-backup-jammy', + 'borg-borg-backup-noble'): homedir = os.path.join('/opt/backups/', username) borg_repo = os.path.join(homedir, 'backup') authorized_keys = os.path.join(homedir, '.ssh', 'authorized_keys') @@ -82,6 +88,11 @@ def test_borg_backup(host): if hostname == 'borg-backup01.region.provider.opendev.org': pytest.skip() + # Note that newer borg (>=1.2) will exit non zero for warnings. This + # is expected and we try to mitigate common problems like files changing + # while backed up by excluding the log file we write to in the command + # from the backups. + # https://borgbackup.readthedocs.io/en/1.2-maint/usage/general.html#return-codes cmd = host.run( '/usr/local/bin/borg-backup borg-backup01.region.provider.opendev.org 2>> ' '/var/log/borg-backup-borg-backup01.region.provider.opendev.org.log') diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index be85e47f84..e91c27248b 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -423,6 +423,8 @@ label: ubuntu-bionic - name: borg-backup-jammy.opendev.org label: ubuntu-jammy + - name: borg-backup-noble.opendev.org + label: ubuntu-noble groups: - <<: *bastion_group vars: @@ -453,6 +455,9 @@ borg-backup-jammy.opendev.org: host_copy_output: '/var/log/borg-backup-borg-backup01.region.provider.opendev.org.log': logs + borg-backup-noble.opendev.org: + host_copy_output: + '/var/log/borg-backup-borg-backup01.region.provider.opendev.org.log': logs - job: name: system-config-run-mirror-base