From 1c069add54a8b51fe42c36dfb2e64389395fd112 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jim@acmegating.com>
Date: Mon, 15 Jul 2024 15:50:49 -0700
Subject: [PATCH] Run zuul-launcher
This doesn't do anything substantial yet, but we'd like to start running
the server soon in order to test it out.
Change-Id: I9eb2bccd6e5e9a064cbaff10676aeb1af6653f98
---
doc/source/zuul.rst | 1 +
hiera/common.yaml | 1 +
inventory/service/groups.yaml | 3 ++
playbooks/roles/zuul-launcher/README.rst | 1 +
.../roles/zuul-launcher/defaults/main.yaml | 1 +
.../zuul-launcher/files/docker-compose.yaml | 15 ++++++
.../roles/zuul-launcher/files/logging.conf | 50 +++++++++++++++++++
.../roles/zuul-launcher/tasks/graceful.yaml | 42 ++++++++++++++++
playbooks/roles/zuul-launcher/tasks/main.yaml | 39 +++++++++++++++
playbooks/roles/zuul-launcher/tasks/pull.yaml | 4 ++
.../roles/zuul-launcher/tasks/start.yaml | 4 ++
playbooks/roles/zuul-launcher/tasks/stop.yaml | 6 +++
playbooks/service-zuul.yaml | 5 ++
playbooks/zuul/run-base.yaml | 1 +
.../group_vars/zuul-launcher.yaml.j2 | 1 +
playbooks/zuul_pull.yaml | 6 +++
playbooks/zuul_reboot.yaml | 24 +++++++++
playbooks/zuul_start.yaml | 6 +++
playbooks/zuul_stop.yaml | 6 +++
testinfra/test_zuul_launcher.py | 25 ++++++++++
zuul.d/system-config-run.yaml | 8 +++
21 files changed, 249 insertions(+)
create mode 100644 playbooks/roles/zuul-launcher/README.rst
create mode 100644 playbooks/roles/zuul-launcher/defaults/main.yaml
create mode 100644 playbooks/roles/zuul-launcher/files/docker-compose.yaml
create mode 100644 playbooks/roles/zuul-launcher/files/logging.conf
create mode 100644 playbooks/roles/zuul-launcher/tasks/graceful.yaml
create mode 100644 playbooks/roles/zuul-launcher/tasks/main.yaml
create mode 100644 playbooks/roles/zuul-launcher/tasks/pull.yaml
create mode 100644 playbooks/roles/zuul-launcher/tasks/start.yaml
create mode 100644 playbooks/roles/zuul-launcher/tasks/stop.yaml
create mode 100644 playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2
create mode 100644 testinfra/test_zuul_launcher.py
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index 4c684220e9..b16eedfb27 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -16,6 +16,7 @@ At a Glance
* zuul*.opendev.org
* ze*.opendev.org
* zm*.opendev.org
+ * zl*.opendev.org
:Configuration:
* :config:`zuul/main.yaml`
* :config:`zuul.d`
diff --git a/hiera/common.yaml b/hiera/common.yaml
index d9faaa7b4c..dcf032a9dc 100644
--- a/hiera/common.yaml
+++ b/hiera/common.yaml
@@ -76,6 +76,7 @@ cacti_hosts:
- zk04.opendev.org
- zk05.opendev.org
- zk06.opendev.org
+- zl01.opendev.org
- zm01.opendev.org
- zm02.opendev.org
- zm03.opendev.org
diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml
index f6dc3419fb..f0d09a1364 100644
--- a/inventory/service/groups.yaml
+++ b/inventory/service/groups.yaml
@@ -183,10 +183,13 @@ groups:
- zuul-lb[0-9]*.opendev.org
zuul:
- ze[0-9]*.opendev.org
+ - zl[0-9]*.opendev.org
- zm[0-9]*.opendev.org
- zuul[0-9]*.opendev.org
zuul-executor:
- ze[0-9]*.opendev.org
+ zuul-launcher:
+ - zl[0-9]*.opendev.org
zuul-merger:
- zm[0-9]*.opendev.org
zuul-preview:
diff --git a/playbooks/roles/zuul-launcher/README.rst b/playbooks/roles/zuul-launcher/README.rst
new file mode 100644
index 0000000000..c32b2b397b
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/README.rst
@@ -0,0 +1 @@
+Run zuul launcher
diff --git a/playbooks/roles/zuul-launcher/defaults/main.yaml b/playbooks/roles/zuul-launcher/defaults/main.yaml
new file mode 100644
index 0000000000..0b7d4ec732
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/defaults/main.yaml
@@ -0,0 +1 @@
+zuul_launcher_start: false
diff --git a/playbooks/roles/zuul-launcher/files/docker-compose.yaml b/playbooks/roles/zuul-launcher/files/docker-compose.yaml
new file mode 100644
index 0000000000..d5e9006ddf
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/files/docker-compose.yaml
@@ -0,0 +1,15 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+ launcher:
+ restart: on-failure
+ image: quay.io/zuul-ci/zuul-launcher:latest
+ network_mode: host
+ user: zuul
+ volumes:
+ - /etc/zuul:/etc/zuul
+ - /home/zuuld:/home/zuul
+ - /var/lib/zuul:/var/lib/zuul
+ - /var/log/zuul:/var/log/zuul
diff --git a/playbooks/roles/zuul-launcher/files/logging.conf b/playbooks/roles/zuul-launcher/files/logging.conf
new file mode 100644
index 0000000000..28031a4371
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/files/logging.conf
@@ -0,0 +1,50 @@
+[loggers]
+keys=root,zuul,gerrit,gear
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_zuul]
+level=DEBUG
+handlers=debug,normal
+qualname=zuul
+
+[logger_gerrit]
+level=INFO
+handlers=debug,normal
+qualname=gerrit
+
+[logger_gear]
+level=WARNING
+handlers=debug,normal
+qualname=gear
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/launcher-debug.log',)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.WatchedFileHandler
+formatter=simple
+args=('/var/log/zuul/launcher.log',)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=
+class=zuul.lib.logutil.MultiLineFormatter
diff --git a/playbooks/roles/zuul-launcher/tasks/graceful.yaml b/playbooks/roles/zuul-launcher/tasks/graceful.yaml
new file mode 100644
index 0000000000..db591c03a4
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/tasks/graceful.yaml
@@ -0,0 +1,42 @@
+- name: Check if Zuul launcher containers are running
+ # It is possible they are stopped due to some external circumstance.
+ # NOTE: docker-compose ps -q reports exited containers unlike docker ps -q
+ command:
+ cmd: docker-compose ps -q
+ chdir: /etc/zuul-launcher
+ become: true
+ become_user: root
+ register: launcher_container_list
+- name: Gracefully stop Zuul Launcher
+ shell:
+ cmd: docker-compose exec -T launcher zuul-launcher stop
+ chdir: /etc/zuul-launcher
+ become: true
+ become_user: root
+ # Only run the docker exec command if a container is running
+ when: launcher_container_list.stdout_lines | length > 0
+ register: zl_graceful
+ failed_when:
+ - zl_graceful.rc != 0
+ # There is a fun race with docker exec and shutting down the
+ # container running the exec. If the container is stopped while
+ # the exec is running then the command in the exec effectively gets
+ # kill -9'd and the docker exec command rc is 137. Since this
+ # should only happen when the container is stopped we proceed with
+ # the overall graceful shutdown.
+ - zl_graceful.rc != 137
+ # If the exec fails because the container is not running we continue.
+ - "'No container found' not in zl_graceful.stderr"
+- name: Wait for Zuul Launcher to stop
+ shell:
+ cmd: docker-compose ps -q | xargs docker wait
+ chdir: /etc/zuul-launcher
+ become: true
+ become_user: root
+ when: launcher_container_list.stdout_lines | length > 0
+- name: Down Zuul Launcher containers
+ shell:
+ cmd: docker-compose down
+ chdir: /etc/zuul-launcher
+ become: true
+ become_user: root
diff --git a/playbooks/roles/zuul-launcher/tasks/main.yaml b/playbooks/roles/zuul-launcher/tasks/main.yaml
new file mode 100644
index 0000000000..bf20699ca2
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/tasks/main.yaml
@@ -0,0 +1,39 @@
+- name: Install logging config
+ copy:
+ src: logging.conf
+ dest: /etc/zuul/launcher-logging.conf
+
+- name: Rotate launcher logs
+ include_role:
+ name: logrotate
+ vars:
+ logrotate_file_name: /var/log/zuul/launcher.log
+
+- name: Rotate launcher debug logs
+ include_role:
+ name: logrotate
+ vars:
+ logrotate_file_name: /var/log/zuul/launcher-debug.log
+
+- name: Make docker-compose directory
+ file:
+ state: directory
+ path: /etc/zuul-launcher
+
+- name: Install docker-compose file
+ copy:
+ src: docker-compose.yaml
+ dest: /etc/zuul-launcher/docker-compose.yaml
+
+- name: Update container images
+ include_tasks: pull.yaml
+
+- name: Start containers
+ include_tasks: start.yaml
+ when: zuul_launcher_start | bool
+
+# We can prune here as it should leave the "latest" tagged images
+# as well as the currently running images.
+- name: Run docker prune to cleanup unneeded images
+ shell:
+ cmd: docker image prune -f
diff --git a/playbooks/roles/zuul-launcher/tasks/pull.yaml b/playbooks/roles/zuul-launcher/tasks/pull.yaml
new file mode 100644
index 0000000000..630a02e88b
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/tasks/pull.yaml
@@ -0,0 +1,4 @@
+- name: Run docker-compose pull
+ shell:
+ cmd: docker-compose pull
+ chdir: /etc/zuul-launcher
diff --git a/playbooks/roles/zuul-launcher/tasks/start.yaml b/playbooks/roles/zuul-launcher/tasks/start.yaml
new file mode 100644
index 0000000000..593d17c042
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/tasks/start.yaml
@@ -0,0 +1,4 @@
+- name: Run docker-compose up
+ shell:
+ cmd: docker-compose up -d
+ chdir: /etc/zuul-launcher
diff --git a/playbooks/roles/zuul-launcher/tasks/stop.yaml b/playbooks/roles/zuul-launcher/tasks/stop.yaml
new file mode 100644
index 0000000000..9e54b666d6
--- /dev/null
+++ b/playbooks/roles/zuul-launcher/tasks/stop.yaml
@@ -0,0 +1,6 @@
+- name: Stop Zuul Launcher
+ shell:
+ cmd: docker-compose down
+ chdir: /etc/zuul-launcher
+ become: true
+ become_user: root
diff --git a/playbooks/service-zuul.yaml b/playbooks/service-zuul.yaml
index 17fe95fb69..02014d7a20 100644
--- a/playbooks/service-zuul.yaml
+++ b/playbooks/service-zuul.yaml
@@ -15,6 +15,11 @@
- install-docker
- zuul
+- hosts: "zuul-launcher:!disabled"
+ name: "Configure zuul launcher"
+ roles:
+ - zuul-launcher
+
- hosts: "zuul-merger:!disabled"
name: "Configure zuul merger"
roles:
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index cde4585646..554365399a 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -136,6 +136,7 @@
- group_vars/zuul-lb.yaml
- group_vars/zuul.yaml
- group_vars/zuul-executor.yaml
+ - group_vars/zuul-launcher.yaml
- group_vars/zuul-merger.yaml
- group_vars/zuul-scheduler.yaml
- group_vars/zuul-web.yaml
diff --git a/playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2 b/playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2
new file mode 100644
index 0000000000..004442b760
--- /dev/null
+++ b/playbooks/zuul/templates/group_vars/zuul-launcher.yaml.j2
@@ -0,0 +1 @@
+zuul_launcher_start: true
diff --git a/playbooks/zuul_pull.yaml b/playbooks/zuul_pull.yaml
index a6dc7fc126..381d6e2b06 100644
--- a/playbooks/zuul_pull.yaml
+++ b/playbooks/zuul_pull.yaml
@@ -16,6 +16,12 @@
name: zuul-merger
tasks_from: pull
+- hosts: 'zuul-launcher:!disabled'
+ tasks:
+ - include_role:
+ name: zuul-launcher
+ tasks_from: pull
+
- hosts: 'zuul-executor:!disabled'
tasks:
- include_role:
diff --git a/playbooks/zuul_reboot.yaml b/playbooks/zuul_reboot.yaml
index 198cc3fee5..66a524469e 100644
--- a/playbooks/zuul_reboot.yaml
+++ b/playbooks/zuul_reboot.yaml
@@ -54,6 +54,30 @@
name: zuul-merger
tasks_from: start
+- hosts: "zuul-launcher:!disabled"
+ name: "Reboot zuul-launchers gracefully one at a time"
+ serial: 1
+ tasks:
+ - name: Gracefully stop the launcher
+ include_role:
+ name: zuul-launcher
+ tasks_from: graceful
+ - name: Upgrade launcher server packages
+ apt:
+ update_cache: yes
+ upgrade: yes
+ register: apt_action
+ # 20 minute wait for unattended-upgrades to complete
+ delay: 30
+ retries: 40
+ until: apt_action is success or 'Failed to lock apt for exclusive operation' not in apt_action.msg
+ - name: Reboot the launcher server
+ reboot:
+ - name: Start the launcher
+ include_role:
+ name: zuul-launcher
+ tasks_from: start
+
# TODO should we do both schedulers with reboots then do the webs without
# reboots?
- hosts: "zuul-scheduler:!disabled"
diff --git a/playbooks/zuul_start.yaml b/playbooks/zuul_start.yaml
index a295778b7c..e46cc5b103 100644
--- a/playbooks/zuul_start.yaml
+++ b/playbooks/zuul_start.yaml
@@ -18,6 +18,12 @@
name: zuul-merger
tasks_from: start
+- hosts: 'zuul-launcher:!disabled'
+ tasks:
+ - include_role:
+ name: zuul-launcher
+ tasks_from: start
+
- hosts: 'zuul-executor:!disabled'
tasks:
- include_role:
diff --git a/playbooks/zuul_stop.yaml b/playbooks/zuul_stop.yaml
index 6f53fdaf79..3944344b01 100644
--- a/playbooks/zuul_stop.yaml
+++ b/playbooks/zuul_stop.yaml
@@ -19,6 +19,12 @@
name: zuul-merger
tasks_from: stop
+- hosts: 'zuul-launcher:!disabled'
+ tasks:
+ - include_role:
+ name: zuul-launcher
+ tasks_from: stop
+
- hosts: 'zuul-executor:!disabled'
tasks:
- include_role:
diff --git a/testinfra/test_zuul_launcher.py b/testinfra/test_zuul_launcher.py
new file mode 100644
index 0000000000..491ed22141
--- /dev/null
+++ b/testinfra/test_zuul_launcher.py
@@ -0,0 +1,25 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import util
+
+testinfra_hosts = ['zl01.opendev.org']
+
+
+def test_iptables(host):
+ rules = util.verify_iptables(host)
+
+ for rule in rules:
+ assert '--dport 79' not in rule
+ assert '--dport 7900' not in rule
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index e1c8b212aa..3066c49aa7 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -962,6 +962,8 @@
label: ubuntu-focal
- name: zm01.opendev.org
label: ubuntu-jammy
+ - name: zl01.opendev.org
+ label: ubuntu-jammy
- name: ze01.opendev.org
label: ubuntu-jammy
- name: zuul02.opendev.org
@@ -990,6 +992,11 @@
'/etc/hosts': logs
'/etc/zuul/zuul.conf': logs
'/var/log/zuul/merger-debug.log': logs
+ zl01.opendev.org:
+ host_copy_output:
+ '/etc/hosts': logs
+ '/etc/zuul/zuul.conf': logs
+ '/var/log/zuul/launcher-debug.log': logs
ze01.opendev.org:
host_copy_output:
'/etc/hosts': logs
@@ -1032,6 +1039,7 @@
- testinfra/test_zuul_executor.py
- testinfra/test_zuul_scheduler.py
- testinfra/test_zuul_merger.py
+ - testinfra/test_zuul_launcher.py
- testinfra/test_zuul_db.py
- testinfra/util.py