diff --git a/playbooks/zuul/run-production-bootstrap-bridge.yaml b/playbooks/zuul/run-production-bootstrap-bridge.yaml index e833fc4514..036acd2c0c 100644 --- a/playbooks/zuul/run-production-bootstrap-bridge.yaml +++ b/playbooks/zuul/run-production-bootstrap-bridge.yaml @@ -3,3 +3,9 @@ - add-bastion-host - import_playbook: ../bootstrap-bridge.yaml + +- name: Wait for child jobs + zuul_return: + data: + zuul: + pause: true diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index 23360b6795..f85aee00dd 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -2,14 +2,59 @@ # in projects.yaml because it's easier to keep an overall view of # what's happening in there. -# Make sure only one run of a system-config playbook happens at a time +# Make sure only one run happens at a time. The deploy pipeline +# should keep things in order, but this is to stop perodic jobs +# jumping in. - semaphore: - name: infra-prod-playbook + name: infra-prod-deployment + max: 1 + +# This semaphore limits the total number of production playbook +# jobs that can run on bridge at one time. We want things to run in +# parallel but we have a lot of jobs (particularly in the periodic +# pipeline) that we don't want to run all at once. +- semaphore: + name: infra-prod-playbook-limit + # TODO(clarkb) this semaphore allows us to stage the rollout of + # parallel infra-prod job exceution in two steps. First we reorganize + # everything but roughly keep the same behaviors as before (max: 1). + # When we are happy with that we can bump this to 2 or higher and see + # things run in parallel. max: 1 +- job: + name: infra-prod-bootstrap-bridge + parent: opendev-infra-prod-setup-src + semaphores: infra-prod-deployment + description: | + Configure the bastion host (bridge) + + This job does minimal configuration on the bastion host + (bridge.openstack.org) to allow it to run system-config + playbooks against our production hosts. It sets up Ansible + and root keys on the host. It also synchronizes the + system-config repo from the executor to the bastion. + + Note that this is separate to infra-prod-service-bridge; + bridge in it's role as the bastion host actaully runs that + against itself; it includes things not strictly needed to make + the host able to deploy system-config. + + This job is the parent of all deployment jobs, and will pause + until they finish. This prevents conflicts between deployment + jobs from changes and periodic runs (which use HEAD of + master). + run: playbooks/zuul/run-production-bootstrap-bridge.yaml + # Do not set file matchers on this job. We must always run this job + # before any other infra-prod jobs to ensure system-config is up to + # date on bridge before we run our playbooks. + nodeset: + nodes: [] + - job: name: infra-prod-playbook - parent: opendev-infra-prod-base + parent: opendev-infra-prod-setup-keys + semaphores: infra-prod-playbook-limit description: | Run specified playbook against productions hosts. @@ -19,7 +64,6 @@ /home/zuul/src/opendev.org/opendev/system-config/playbooks on the bastion host. abstract: true - semaphores: infra-prod-playbook run: playbooks/zuul/run-production-playbook.yaml post-run: playbooks/zuul/run-production-playbook-post.yaml required-projects: @@ -30,41 +74,12 @@ infra_prod_playbook_encrypt_log: true nodeset: nodes: [] - -- job: - name: infra-prod-bootstrap-bridge - parent: opendev-infra-prod-setup-src - description: | - Configure the bastion host (bridge) - - This job does minimal configuration on the bastion host - (bridge.openstack.org) to allow it to run system-config - playbooks against our production hosts. It sets up Ansible - and root keys on the host. It also synchronizes the system-config - repo from the executor to the bastion. This is necessary to - emit an up to date known_hosts file when adding new hosts to - the inventory. - - Note that this is separate to infra-prod-service-bridge; - bridge in it's role as the bastion host actaully runs that - against itself; it includes things not strictly needed to make - the host able to deploy system-config. - # While we don't run the infra-prod-playbook in this job we do run - # system-config git repo updates. Until we're ready to stop running - # system-config updates in every job we use this semaphore to ensure - # exclusivity. - semaphores: infra-prod-playbook - run: playbooks/zuul/run-production-bootstrap-bridge.yaml - files: - - playbooks/bootstrap-bridge.yaml - - playbooks/zuul/run-production-bootstrap-bridge.yaml - - playbooks/zuul/run-production-bootstrap-bridge-add-rootkey.yaml - - playbooks/roles/install-ansible/ - - playbooks/roles/root-keys/ - - inventory/base/hosts.yaml - - inventory/service/group_vars/bastion.yaml - nodeset: - nodes: [] + dependencies: + - name: infra-prod-bootstrap-bridge + # This is a hard dependency because we require the bootstrap job to + # have run before we start any playbook jobs, otherwise our buildset + # would not hold the bridge semaphore and we may not have the correct + # system-config state on bridge. - job: name: infra-prod-base diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 3c386fa484..67c6270b7f 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -340,7 +340,10 @@ # NOTE: infra-prod-* jobs have a hierarchy below that ensure # they can run in parallel. We are deliberately keeping their # dependencies here rather than job definitions to help keep - # these relationships clear. + # these relationships clear. The one exception to this is the + # base infra-prod-playbook job depends on infra-prod-bootstrap-bridge. + # We make this exception because it is vital that bootstrap-bridge + # run before everything else always. # This installs the ansible on bridge that all the infra-prod # jobs will run with. Note the jobs use this ansible to then @@ -348,10 +351,7 @@ - infra-prod-bootstrap-bridge # From now on, all jobs should depend on base - - infra-prod-base: &infra-prod-base - dependencies: - - name: infra-prod-bootstrap-bridge - soft: true + - infra-prod-base # Legacy puppet hosts - infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else @@ -635,7 +635,7 @@ # Nightly runs of ansible things for catchup # Keep in order from above - infra-prod-bootstrap-bridge - - infra-prod-base: *infra-prod-base + - infra-prod-base - infra-prod-remote-puppet-else: *infra-prod-remote-puppet-else - infra-prod-letsencrypt: *infra-prod-letsencrypt - infra-prod-service-bridge: *infra-prod-service-bridge