system-config/zuul.d/project.yaml
Ian Wienand d616ec9d9a Bootstrap-bridge as top-level job
The idea here is

 * all prod jobs are parented to the boostrap-bridge job (they have a
   hard dependency on this job).

 * the bootstrap-bridge job checks out the system-config source to the
   right place (the commit for a change, master HEAD for periodic). This
   was actually implemented in a prior change. We're just taking full
   advantage of it here.

 * bootstrap-bridge pauses once bridge is setup to the right place

 * the child jobs now don't have to worry about cloning system-config;
   they can be sure that it's at the right place for them.  they just
   need keys so their executor can log into bridge and run the
   playbooks against the production hosts

 * the bootstrap-bridge job is paused with a semaphore stopping any
   other runs jumping in.  in deployment, zuul is ordering it for us
   anyway.  so really this is stopping conflicts with the periodic
   jobs.

 * in theory - all the child production jobs could run in parallel
   while the boostrap jobs waits for them (modulo dependencies they
   have expressed; e.g. needing letsencyrpt or backup jobs to have
   run). To begin with we limit this with a second semaphore with a
   limit of 1. We can roll this out and check things mostly operate
   as they did before then bump the max value on this semaphore upwards
   to run things in parallel.

 * does this work?  I have no idea :) it seems difficult to test
   outside production because in the testing side everything is its
   own little world, there's no overarching bootstrap job.

Depends-On: https://review.opendev.org/c/opendev/base-jobs/+/942740
Change-Id: I7d2c4737f900c9b964855c4d03ca58a2de2d60b3
2025-02-25 14:17:14 -08:00

682 lines
28 KiB
YAML

- project:
templates:
- system-config-zuul-role-integration
- system-config-gerrit-images
check:
jobs:
- opendev-tox-docs
- opendev-buildset-registry
- tox-linters
- system-config-run-base
- system-config-run-base-ansible-devel:
voting: false
- system-config-run-borg-backup
- system-config-run-dns
- system-config-run-eavesdrop:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-accessbot
soft: true
- name: system-config-build-image-ircbot
soft: true
- name: system-config-build-image-matrix-eavesdrop
soft: true
- system-config-run-codesearch:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-hound
soft: true
- system-config-run-kerberos
- system-config-run-lists3:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-mailman
soft: true
- system-config-run-nodepool:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-zookeeper-statsd
soft: true
- system-config-run-meetpad
- system-config-run-mirror-x86
- system-config-run-mirror-update
- system-config-run-paste:
dependencies:
- name: opendev-buildset-registry
- system-config-run-static
- system-config-run-docker-registry
- system-config-run-etherpad:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-etherpad
soft: true
- system-config-run-gitea:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-gitea
soft: true
- name: system-config-build-image-haproxy-statsd
soft: true
- system-config-run-grafana:
dependencies:
- name: opendev-buildset-registry
- system-config-run-graphite
- system-config-run-keycloak
- system-config-run-review-3.10:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-gerrit-3.10
soft: true
- system-config-run-review-3.11:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-gerrit-3.11
soft: true
- system-config-upgrade-review:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-gerrit-3.10
soft: true
- name: system-config-build-image-gerrit-3.11
soft: true
- system-config-build-image-refstack
- system-config-run-refstack:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-refstack
soft: true
- system-config-run-tracing
- system-config-run-zookeeper:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-zookeeper-statsd
soft: true
- system-config-run-zuul:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-zookeeper-statsd
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-build-image-assets
- system-config-build-image-jinja-init:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-base-3.11-bookworm
soft: true
- system-config-build-image-gitea-init:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-jinja-init
soft: true
- system-config-build-image-hound:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-base-3.11-bookworm
soft: true
- system-config-build-image-etherpad
- system-config-build-image-mailman
- system-config-build-image-gitea:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-assets
soft: true
- system-config-build-image-haproxy-statsd:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-base-3.11-bookworm
soft: true
- system-config-build-image-zookeeper-statsd:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-base-3.11-bookworm
soft: true
- system-config-build-image-accessbot:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-base-3.11-bookworm
soft: true
- system-config-build-image-ircbot:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-builder-3.11-bookworm
soft: true
- system-config-build-image-matrix-eavesdrop:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-builder-3.11-bookworm
soft: true
- system-config-build-image-python-base-3.10-bookworm
- system-config-build-image-python-base-3.11-bookworm
- system-config-build-image-python-base-3.11-bookworm-debug
- system-config-build-image-python-base-3.12-bookworm
- system-config-build-image-python-base-3.12-bookworm-debug
- system-config-build-image-python-builder-3.10-bookworm
- system-config-build-image-python-builder-3.11-bookworm
- system-config-build-image-python-builder-3.12-bookworm
- system-config-build-image-uwsgi-base-3.10-bookworm
- system-config-build-image-uwsgi-base-3.11-bookworm
- system-config-build-image-uwsgi-base-3.12-bookworm
check-arm64:
jobs:
- system-config-run-base-arm64
- system-config-run-mirror-arm64
gate:
jobs:
- opendev-tox-docs
- opendev-buildset-registry
- tox-linters
- system-config-run-base
- system-config-run-borg-backup
- system-config-run-dns
- system-config-run-eavesdrop:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-accessbot
soft: true
- name: system-config-upload-image-ircbot
soft: true
- name: system-config-upload-image-matrix-eavesdrop
soft: true
- system-config-run-codesearch:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-hound
soft: true
- system-config-run-kerberos
- system-config-run-lists3:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-mailman
soft: true
- system-config-run-nodepool:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-zookeeper-statsd
soft: true
- system-config-run-meetpad
- system-config-run-mirror-x86
- system-config-run-mirror-update
- system-config-run-paste:
dependencies:
- name: opendev-buildset-registry
- system-config-run-static
- system-config-run-docker-registry
- system-config-run-etherpad:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-etherpad
soft: true
- system-config-run-gitea:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-gitea
soft: true
- name: system-config-upload-image-haproxy-statsd
soft: true
- system-config-run-grafana:
dependencies:
- name: opendev-buildset-registry
- system-config-run-graphite
- system-config-run-keycloak
- system-config-run-review-3.10:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-gerrit-3.10
soft: true
- system-config-run-review-3.11:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-gerrit-3.11
soft: true
- system-config-run-refstack:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-refstack
soft: true
- system-config-run-tracing
- system-config-run-zookeeper:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-zookeeper-statsd
soft: true
- system-config-run-zuul:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-zookeeper-statsd
soft: true
- system-config-run-zuul-preview
- system-config-run-letsencrypt
- system-config-upload-image-jinja-init:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-python-base-3.11-bookworm
soft: true
- system-config-upload-image-gitea-init:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-jinja-init
soft: true
- system-config-upload-image-hound:
dependencies:
- name: opendev-buildset-registry
- name: system-config-build-image-python-base-3.11-bookworm
soft: true
- system-config-upload-image-assets
- system-config-upload-image-etherpad
- system-config-upload-image-mailman
- system-config-upload-image-gitea:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-assets
soft: true
- system-config-upload-image-refstack
- system-config-upload-image-haproxy-statsd:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-python-base-3.11-bookworm
soft: true
- system-config-upload-image-zookeeper-statsd:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-python-base-3.11-bookworm
soft: true
- system-config-upload-image-accessbot:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-python-base-3.11-bookworm
soft: true
- system-config-upload-image-ircbot:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-python-builder-3.11-bookworm
soft: true
- system-config-upload-image-matrix-eavesdrop:
dependencies:
- name: opendev-buildset-registry
- name: system-config-upload-image-python-builder-3.11-bookworm
soft: true
- system-config-upload-image-python-base-3.10-bookworm
- system-config-upload-image-python-base-3.11-bookworm
- system-config-upload-image-python-base-3.11-bookworm-debug
- system-config-upload-image-python-base-3.12-bookworm
- system-config-upload-image-python-base-3.12-bookworm-debug
- system-config-upload-image-python-builder-3.10-bookworm
- system-config-upload-image-python-builder-3.11-bookworm
- system-config-upload-image-python-builder-3.12-bookworm
- system-config-upload-image-uwsgi-base-3.10-bookworm
- system-config-upload-image-uwsgi-base-3.11-bookworm
- system-config-upload-image-uwsgi-base-3.12-bookworm
promote:
jobs:
- opendev-promote-docs
deploy:
jobs:
- system-config-promote-image-assets
- system-config-promote-image-hound
- system-config-promote-image-jinja-init
- system-config-promote-image-gitea-init
- system-config-promote-image-gitea
- system-config-promote-image-etherpad
- system-config-promote-image-mailman
- system-config-promote-image-haproxy-statsd
- system-config-promote-image-zookeeper-statsd
- system-config-promote-image-accessbot
- system-config-promote-image-refstack
- system-config-promote-image-ircbot
- system-config-promote-image-matrix-eavesdrop
- system-config-promote-image-python-base-3.10-bookworm
- system-config-promote-image-python-base-3.11-bookworm
- system-config-promote-image-python-base-3.11-bookworm-debug
- system-config-promote-image-python-base-3.12-bookworm
- system-config-promote-image-python-base-3.12-bookworm-debug
- system-config-promote-image-python-builder-3.10-bookworm
- system-config-promote-image-python-builder-3.11-bookworm
- system-config-promote-image-python-builder-3.12-bookworm
- system-config-promote-image-uwsgi-base-3.10-bookworm
- system-config-promote-image-uwsgi-base-3.11-bookworm
- system-config-promote-image-uwsgi-base-3.12-bookworm
# NOTE: infra-prod-* jobs have a hierarchy below that ensure
# they can run in parallel. We are deliberately keeping their
# dependencies here rather than job definitions to help keep
# these relationships clear. The one exception to this is the
# base infra-prod-playbook job depends on infra-prod-bootstrap-bridge.
# We make this exception because it is vital that bootstrap-bridge
# run before everything else always.
# This installs the ansible on bridge that all the infra-prod
# jobs will run with. Note the jobs use this ansible to then
# run against zuul's checkout of system-config.
- infra-prod-bootstrap-bridge
# From now on, all jobs should depend on base
- infra-prod-base
# Legacy puppet hosts
- infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else
dependencies:
- name: infra-prod-base
soft: true
#
# Only depends on base, or amongst themselves.
#
- infra-prod-service-bridge: &infra-prod-service-bridge
dependencies:
- name: infra-prod-base
soft: true
- infra-prod-run-cloud-launcher: &infra-prod-run-cloud-launcher
dependencies:
# depends on the cloud config written out by
# service-bridge
- name: infra-prod-service-bridge
soft: true
- infra-prod-service-kerberos: &infra-prod-service-kerberos
dependencies:
- name: infra-prod-base
soft: true
- infra-prod-service-afs: &infra-prod-service-afs
dependencies:
- name: infra-prod-base
soft: true
# NOTE(ianw) in theory we'd want auth changes before
# updating services like openafs using them. Not sure
# in practice this matters much; we very rarely change
# things here anyway.
- name: infra-prod-service-kerberos
soft: true
- infra-prod-service-nameserver: &infra-prod-service-nameserver
dependencies:
- name: infra-prod-base
soft: true
- infra-prod-service-mirror-update: &infra-prod-service-mirror-update
dependencies:
- name: infra-prod-base
soft: true
#
# Only depends on base, or image promotion.
#
- infra-prod-service-gitea-lb: &infra-prod-service-gitea-lb
dependencies:
- name: infra-prod-base
soft: true
- name: system-config-promote-image-haproxy-statsd
soft: true
- infra-prod-service-zuul-db: &infra-prod-service-zuul-db
dependencies:
- name: infra-prod-base
soft: true
- infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb
dependencies:
- name: infra-prod-base
soft: true
- name: system-config-promote-image-haproxy-statsd
soft: true
#
# Hosts using certificates and backups
#
# Hosts that backup should depend on this as this will create
# the users and deploy the keys required for the borg-backup
# role to work.
- infra-prod-service-borg-backup: &infra-prod-service-borg-backup
dependencies:
- name: infra-prod-base
soft: true
# Hosts that have letsencrypt certs should depend on this, as
# it will write out the key material before they try to start
# services that depend on it. For simplicity, we parent to
# this job.
- infra-prod-letsencrypt: &infra-prod-letsencrypt
dependencies:
- name: infra-prod-base
soft: true
- name: infra-prod-service-nameserver
soft: true
# letsencrypt depdencies. keep in alphabetical order
- infra-prod-service-codesearch: &infra-prod-service-codesearch
dependencies:
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-hound
soft: true
- infra-prod-service-eavesdrop: &infra-prod-service-eavesdrop
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-ircbot
soft: true
- name: system-config-promote-image-matrix-eavesdrop
soft: true
- infra-prod-service-etherpad: &infra-prod-service-etherpad
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-etherpad
soft: true
- infra-prod-service-gitea: &infra-prod-service-gitea
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-gitea
soft: true
- infra-prod-service-grafana: &infra-prod-service-grafana
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-graphite: &infra-prod-service-graphite
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-keycloak: &infra-prod-service-keycloak
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-meetpad: &infra-prod-service-meetpad
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-lists3: &infra-prod-service-lists3
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-mailman
soft: true
- infra-prod-service-mirror: &infra-prod-service-mirror
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-nodepool: &infra-prod-service-nodepool
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-static: &infra-prod-service-static
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-paste: &infra-prod-service-paste
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-registry: &infra-prod-service-registry
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-refstack: &infra-prod-service-refstack
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-refstack
soft: true
- infra-prod-service-review: &infra-prod-service-review
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-gerrit-3.10
soft: true
- infra-prod-service-tracing: &infra-prod-service-tracing
dependencies:
- name: infra-prod-letsencrypt
soft: true
- infra-prod-service-zookeeper: &infra-prod-service-zookeeper
dependencies:
- name: infra-prod-letsencrypt
soft: true
- name: system-config-promote-image-zookeeper-statsd
soft: true
- infra-prod-service-zuul: &infra-prod-service-zuul
dependencies:
- name: infra-prod-service-borg-backup
soft: true
- name: infra-prod-letsencrypt
soft: true
# should reconfigure after any project updates
- name: infra-prod-manage-projects
soft: true
- infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview
dependencies:
- name: infra-prod-letsencrypt
soft: true
#
# Jobs that run as secondary steps
#
# accessbot should run on a setup eavesdrop host
- infra-prod-run-accessbot: &infra-prod-run-accessbot
dependencies:
- name: infra-prod-base
soft: true
- name: infra-prod-service-eavesdrop
soft: true
- name: system-config-promote-image-accessbot
soft: true
# manage-projects runs jeepyb etc. and should run on
# a setup review host. also sets up gitea
- infra-prod-manage-projects: &infra-prod-manage-projects
dependencies:
- name: infra-prod-base
soft: true
- name: infra-prod-service-review
soft: true
- name: infra-prod-service-gitea
soft: true
- name: system-config-promote-image-gerrit-3.10
soft: true
# Note that this job also runs from project-config, so we
# match system-config specific files here rather than the
# job definition.
files:
- inventory/.*
- playbooks/manage-projects.yaml
- inventory/service/group_vars/review.yaml
- inventory/service/group_vars/gitea.yaml
- inventory/service/host_vars/gitea
- inventory/service/host_vars/review
- playbooks/roles/gitea-git-repos/
- playbooks/roles/gerrit/defaults/main.yaml
- playbooks/roles/gerrit/tasks/manage-projects.yaml
periodic:
jobs:
- developer-openstack-goaccess-report
- docs-opendev-goaccess-report
- docs-openstack-goaccess-report
- docs-starlingx-goaccess-report
- governance-openstack-goaccess-report
- releases-openstack-goaccess-report
- security-openstack-goaccess-report
- specs-openstack-goaccess-report
- tarballs-opendev-goaccess-report
- zuul-ci-goaccess-report
# Image mirror jobs
- opendev-mirror-container-image-buildkit
- opendev-mirror-container-image-gerrit
- opendev-mirror-container-image-golang
- opendev-mirror-container-image-httpd
- opendev-mirror-container-image-haproxy
- opendev-mirror-container-image-jaegertracing
- opendev-mirror-container-image-mariadb
- opendev-mirror-container-image-multiarch-qemu-user-static
- opendev-mirror-container-image-node
- opendev-mirror-container-image-python-base
- opendev-mirror-container-image-python-builder
- opendev-mirror-container-image-uwsgi-base
- opendev-mirror-container-image-registry
- opendev-mirror-container-image-ubuntu
- opendev-mirror-container-image-zookeeper
# Nightly runs of ansible things for catchup
# Keep in order from above
- infra-prod-bootstrap-bridge
- infra-prod-base
- infra-prod-remote-puppet-else: *infra-prod-remote-puppet-else
- infra-prod-letsencrypt: *infra-prod-letsencrypt
- infra-prod-service-bridge: *infra-prod-service-bridge
- infra-prod-run-cloud-launcher: *infra-prod-run-cloud-launcher
- infra-prod-service-kerberos: *infra-prod-service-kerberos
- infra-prod-service-afs: *infra-prod-service-afs
- infra-prod-service-nameserver: *infra-prod-service-nameserver
- infra-prod-service-mirror-update: *infra-prod-service-mirror-update
- infra-prod-service-gitea-lb: *infra-prod-service-gitea-lb
- infra-prod-service-zuul-db: *infra-prod-service-zuul-db
- infra-prod-service-zuul-lb: *infra-prod-service-zuul-lb
- infra-prod-service-borg-backup: *infra-prod-service-borg-backup
- infra-prod-letsencrypt: *infra-prod-letsencrypt
- infra-prod-service-codesearch: *infra-prod-service-codesearch
- infra-prod-service-eavesdrop: *infra-prod-service-eavesdrop
- infra-prod-service-etherpad: *infra-prod-service-etherpad
- infra-prod-service-gitea: *infra-prod-service-gitea
- infra-prod-service-grafana: *infra-prod-service-grafana
- infra-prod-service-graphite: *infra-prod-service-graphite
- infra-prod-service-keycloak: *infra-prod-service-keycloak
- infra-prod-service-meetpad: *infra-prod-service-meetpad
- infra-prod-service-lists3: *infra-prod-service-lists3
- infra-prod-service-mirror: *infra-prod-service-mirror
- infra-prod-service-nodepool: *infra-prod-service-nodepool
- infra-prod-service-static: *infra-prod-service-static
- infra-prod-service-paste: *infra-prod-service-paste
- infra-prod-service-registry: *infra-prod-service-registry
- infra-prod-service-refstack: *infra-prod-service-refstack
- infra-prod-service-review: *infra-prod-service-review
- infra-prod-service-tracing: *infra-prod-service-tracing
- infra-prod-service-zookeeper: *infra-prod-service-zookeeper
- infra-prod-service-zuul: *infra-prod-service-zuul
- infra-prod-service-zuul-preview: *infra-prod-service-zuul-preview
- infra-prod-run-accessbot: *infra-prod-run-accessbot
- infra-prod-manage-projects: *infra-prod-manage-projects
opendev-prod-hourly:
jobs:
- infra-prod-bootstrap-bridge
- infra-prod-service-bridge: *infra-prod-service-bridge
- infra-prod-service-nodepool: *infra-prod-service-nodepool
- infra-prod-service-registry: *infra-prod-service-registry
- infra-prod-service-zuul: *infra-prod-service-zuul
- infra-prod-service-eavesdrop: *infra-prod-service-eavesdrop