From aad8144bfb70cc0a46b3fc6238e7e8029dce29bd Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Sat, 17 Jun 2017 08:36:47 +0100 Subject: [PATCH] Implement serialisable rolling upgrade The current rolling upgrade implementation relies on the role to orchestrate the rolling upgrade. When the role is executed using playbook serialisation, the db sync contract is executed before all hosts are upgraded, potentially resulting in data corruption. This patch returns the role to the simpler, best practice model of expecting that the role is applied to a single host and that the playbook handles orchestration. This method can be used with any form of serialisation. Depends-On: Ie90cdcbf9e73082a2074c8832b7490d188e178af Change-Id: I5650f16b9a115bd392012b743788057a94d09226 --- handlers/main.yml | 71 ++---------------- tasks/keystone_apache.yml | 27 +++---- tasks/keystone_cleanup_old_facts.yml | 14 ++-- tasks/keystone_db_setup.yml | 73 +++++++++++++------ tasks/keystone_federation_sp_setup.yml | 12 +-- tasks/keystone_idp_metadata.yml | 5 +- tasks/keystone_idp_self_signed_create.yml | 3 +- tasks/keystone_idp_self_signed_distribute.yml | 3 +- tasks/keystone_init_systemd.yml | 12 +-- tasks/keystone_install.yml | 31 ++++---- tasks/keystone_ldap_setup.yml | 12 +-- tasks/keystone_nginx.yml | 12 +-- tasks/keystone_post_install.yml | 18 ++--- tasks/keystone_ssl_key_create.yml | 6 +- tasks/keystone_ssl_key_distribute.yml | 9 +-- tasks/keystone_ssl_user_provided.yml | 9 +-- tasks/keystone_uwsgi.yml | 3 +- tasks/main.yml | 19 ++--- 18 files changed, 125 insertions(+), 214 deletions(-) diff --git a/handlers/main.yml b/handlers/main.yml index 8998c833..64752414 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -13,26 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Restart web server on first node - command: "/bin/true" - notify: - - Restart web server - - Wait for web server to complete starting - when: - - inventory_hostname == groups['keystone_all'][0] - tags: - - keystone-config - -- name: Restart web server on other nodes - command: "/bin/true" - notify: - - Restart web server - - Wait for web server to complete starting - when: - - inventory_hostname != groups['keystone_all'][0] - tags: - - keystone-config - - name: Restart web server service: name: "{{ (keystone_apache_enabled | bool) | ternary(keystone_system_service_name, 'nginx') }}" @@ -43,8 +23,6 @@ until: _restart | success retries: 5 delay: 2 - tags: - - keystone-config - name: Wait for web server to complete starting wait_for: @@ -57,32 +35,7 @@ register: _wait_check until: _wait_check | success retries: 5 - tags: - - keystone-config - -- name: Restart uWSGI on first node - command: "/bin/true" - when: - - inventory_hostname == groups['keystone_all'][0] - notify: - - Stop uWSGI - - Copy new policy file into place - - Start uWSGI - - Wait for uWSGI socket to be ready - tags: - - keystone-config - -- name: Restart uWSGI on other nodes - command: "/bin/true" - when: - - inventory_hostname != groups['keystone_all'][0] - notify: - - Stop uWSGI - - Copy new policy file into place - - Start uWSGI - - Wait for uWSGI socket to be ready - tags: - - keystone-config + listen: "Restart web server" - name: Stop uWSGI service: @@ -96,8 +49,7 @@ with_items: "{{ keystone_wsgi_program_names }}" when: - not keystone_mod_wsgi_enabled | bool - tags: - - keystone-config + listen: "Restart uWSGI" # Note (odyssey4me): # The policy.json file is currently read continually by the services @@ -115,8 +67,7 @@ group: "{{ keystone_system_group_name }}" mode: "0640" remote_src: yes - tags: - - keystone-config + listen: "Restart uWSGI" - name: Start uWSGI service: @@ -131,8 +82,7 @@ with_items: "{{ keystone_wsgi_program_names }}" when: - not keystone_mod_wsgi_enabled | bool - tags: - - keystone-config + listen: "Restart uWSGI" - name: Wait for uWSGI socket to be ready wait_for: @@ -147,8 +97,7 @@ register: _wait_check until: _wait_check | success retries: 5 - tags: - - keystone-config + listen: "Restart uWSGI" - name: Restart Shibd service: @@ -160,13 +109,3 @@ until: _restart | success retries: 5 delay: 2 - tags: - - keystone-config - -- name: Perform a Keystone DB sync contract - command: "{{ keystone_bin }}/keystone-manage db_sync --contract" - become: yes - become_user: "{{ keystone_system_user_name }}" - tags: - - keystone-config - diff --git a/tasks/keystone_apache.yml b/tasks/keystone_apache.yml index a5e77ea3..ba7ff6ac 100644 --- a/tasks/keystone_apache.yml +++ b/tasks/keystone_apache.yml @@ -53,8 +53,7 @@ when: - ansible_pkg_mgr == 'apt' notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server ## NOTE(andymccr): ## We need to enable a module for httpd on RedHat/CentOS using LoadModule inside conf files @@ -66,8 +65,7 @@ when: - ansible_pkg_mgr == 'yum' notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Drop apache2 config files template: @@ -77,8 +75,7 @@ group: "root" with_items: "{{ keystone_apache_configs }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Disable default apache site file: @@ -86,8 +83,7 @@ state: "absent" with_items: "{{ keystone_apache_default_sites }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Enabled keystone vhost file: @@ -98,16 +94,14 @@ - keystone_apache_site_available is defined - keystone_apache_site_enabled is defined notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Ensure Apache ServerName lineinfile: dest: "{{ keystone_apache_conf }}" line: "ServerName {{ ansible_hostname }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Ensure Apache ServerTokens lineinfile: @@ -115,8 +109,7 @@ regexp: '^ServerTokens' line: "ServerTokens {{ keystone_apache_servertokens }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Ensure Apache ServerSignature lineinfile: @@ -124,8 +117,7 @@ regexp: '^ServerSignature' line: "ServerSignature {{ keystone_apache_serversignature }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Remove Listen from Apache config lineinfile: @@ -134,5 +126,4 @@ backrefs: yes line: '#\1' notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server diff --git a/tasks/keystone_cleanup_old_facts.yml b/tasks/keystone_cleanup_old_facts.yml index 75aa3456..9ba40583 100644 --- a/tasks/keystone_cleanup_old_facts.yml +++ b/tasks/keystone_cleanup_old_facts.yml @@ -16,18 +16,18 @@ # TODO(odyssey4me): # This file and the task calling it may be removed in Queens. -- name: Convert the old fact format to the new fact format - ini_file: - dest: "/etc/ansible/facts.d/openstack_ansible.fact" - section: keystone - option: need_db_sync - value: "{{ ansible_local['keystone']['general']['need_db_sync'] | bool }}" - - name: Remove the old fact file file: path: "/etc/ansible/facts.d/keystone.fact" state: absent +- name: Remove the old db_sync fact + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: keystone + option: "need_db_sync" + state: absent + - name: refresh local facts setup: filter: ansible_local diff --git a/tasks/keystone_db_setup.yml b/tasks/keystone_db_setup.yml index e8316303..f28375b2 100644 --- a/tasks/keystone_db_setup.yml +++ b/tasks/keystone_db_setup.yml @@ -13,43 +13,68 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Ensure keystone service stopped on first node - service: - name: "{{ item }}" - state: stopped - register: keystone_stop - failed_when: - - keystone_stop.msg is defined - - "'no service or tool' not in keystone_stop.msg" - - "'systemd could not find' not in keystone_stop.msg" - - "'Could not find the requested service' not in keystone_stop.msg" - with_items: - - "{{ keystone_wsgi_program_names }}" - - "{{ keystone_system_service_name }}" - - name: Check current state of Keystone DB command: "{{ keystone_bin }}/keystone-manage db_sync --check" register: keystone_db_sync_check failed_when: "keystone_db_sync_check.rc == 1" changed_when: "keystone_db_sync_check.rc not in [2, 3, 4]" +- name: Set the db sync local facts + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: keystone + option: "{{ item.name }}" + value: "{{ item.state }}" + with_items: + - name: "need_db_expand" + state: "{{ (keystone_db_sync_check.rc | int == 2) | bool }}" + - name: "need_db_migrate" + state: "{{ (keystone_db_sync_check.rc | int in [2, 3] ) | bool }}" + - name: "need_db_contract" + state: "{{ (keystone_db_sync_check.rc | int in [2, 3, 4]) | bool }}" + +- name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - keystone-config + +- name: Ensure keystone service is stopped + service: + name: "{{ item }}" + state: stopped + register: keystone_stop + failed_when: + - "keystone_stop.msg is defined" + - "'no service or tool' not in keystone_stop.msg" + - "'systemd could not find' not in keystone_stop.msg" + - "'Could not find the requested service' not in keystone_stop.msg" + with_items: + - "{{ keystone_wsgi_program_names }}" + - "{{ keystone_system_service_name }}" + when: + - "(ansible_local['openstack_ansible']['keystone']['need_db_expand'] | bool) or + (ansible_local['openstack_ansible']['keystone']['need_db_migrate'] | bool)" + - name: Perform a Keystone DB sync expand command: "{{ keystone_bin }}/keystone-manage db_sync --expand" become: yes become_user: "{{ keystone_system_user_name }}" - when: "keystone_db_sync_check.rc == 2" + when: + - "ansible_local['openstack_ansible']['keystone']['need_db_expand'] | bool" - name: Perform a Keystone DB sync migrate command: "{{ keystone_bin }}/keystone-manage db_sync --migrate" become: yes become_user: "{{ keystone_system_user_name }}" - when: "keystone_db_sync_check.rc in [2, 3]" - notify: - - Perform a Keystone DB sync contract + when: + - "ansible_local['openstack_ansible']['keystone']['need_db_migrate'] | bool" -- name: Check if Keystone DB sync contract is required - command: "true" - changed_when: "keystone_db_sync_check.rc == 4" - register: dbsync - notify: - - Perform a Keystone DB sync contract +- name: Perform a Keystone DB sync contract + command: "{{ keystone_bin }}/keystone-manage db_sync --contract" + become: yes + become_user: "{{ keystone_system_user_name }}" + when: + - "(keystone_all_software_updated | default('no')) | bool" + - "ansible_local['openstack_ansible']['keystone']['need_db_contract'] | bool" diff --git a/tasks/keystone_federation_sp_setup.yml b/tasks/keystone_federation_sp_setup.yml index dc5931f5..a5129ad7 100644 --- a/tasks/keystone_federation_sp_setup.yml +++ b/tasks/keystone_federation_sp_setup.yml @@ -33,8 +33,7 @@ changed_when: false when: inventory_hostname == groups['keystone_all'][0] notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - Restart Shibd - name: Store sp cert @@ -66,8 +65,7 @@ mode: "0640" when: inventory_hostname != groups['keystone_all'][0] notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - Restart Shibd - name: Distribute sp cert @@ -79,8 +77,7 @@ mode: "0640" when: inventory_hostname != groups['keystone_all'][0] notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - Restart Shibd - name: Set appropriate file ownership on the Shibboleth SP key-pair @@ -93,6 +90,5 @@ - "/etc/shibboleth/sp-key.pem" when: inventory_hostname != groups['keystone_all'][0] notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - Restart Shibd diff --git a/tasks/keystone_idp_metadata.yml b/tasks/keystone_idp_metadata.yml index 4265cdaf..f7ed648e 100644 --- a/tasks/keystone_idp_metadata.yml +++ b/tasks/keystone_idp_metadata.yml @@ -20,7 +20,4 @@ become_user: "{{ keystone_system_user_name }}" when: keystone_idp != {} notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI diff --git a/tasks/keystone_idp_self_signed_create.yml b/tasks/keystone_idp_self_signed_create.yml index de1bbb94..014809b4 100644 --- a/tasks/keystone_idp_self_signed_create.yml +++ b/tasks/keystone_idp_self_signed_create.yml @@ -33,8 +33,7 @@ when: > inventory_hostname == groups['keystone_all'][0] notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Set appropriate file ownership on the IdP self-signed cert file: diff --git a/tasks/keystone_idp_self_signed_distribute.yml b/tasks/keystone_idp_self_signed_distribute.yml index 159a2b3c..cb8b4b6b 100644 --- a/tasks/keystone_idp_self_signed_distribute.yml +++ b/tasks/keystone_idp_self_signed_distribute.yml @@ -30,8 +30,7 @@ retries: 5 delay: 2 notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Set appropriate file ownership on the IdP self-signed cert file: diff --git a/tasks/keystone_init_systemd.yml b/tasks/keystone_init_systemd.yml index 6b3a3920..b1ba7279 100644 --- a/tasks/keystone_init_systemd.yml +++ b/tasks/keystone_init_systemd.yml @@ -32,10 +32,8 @@ owner: "root" group: "root" notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server - name: Place the systemd init script config_template: @@ -47,7 +45,5 @@ config_overrides: "{{ keystone_uwsgi_init_overrides }}" config_type: "ini" notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server diff --git a/tasks/keystone_install.yml b/tasks/keystone_install.yml index a314287a..fd1c81fa 100644 --- a/tasks/keystone_install.yml +++ b/tasks/keystone_install.yml @@ -60,8 +60,7 @@ retries: 5 delay: 2 notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Create developer mode constraint file copy: @@ -120,10 +119,8 @@ copy: "no" when: keystone_get_venv | changed notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server - name: Install pip packages pip: @@ -141,10 +138,8 @@ delay: 2 when: keystone_get_venv | failed or keystone_get_venv | skipped notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server - name: CentOS remove python from path first file: @@ -171,15 +166,21 @@ - src: "{{ keystone_bin }}/keystone-wsgi-public" dest: main notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server -- name: Record the need for a db sync +- name: Initialise the db sync local facts ini_file: dest: "/etc/ansible/facts.d/openstack_ansible.fact" section: keystone - option: need_db_sync - value: True + option: "{{ item.name }}" + value: "{{ item.state }}" + with_items: + - name: "need_db_expand" + state: "True" + - name: "need_db_migrate" + state: "True" + - name: "need_db_contract" + state: "True" when: keystone_get_venv | changed or keystone_venv_dir | changed or install_packages | changed diff --git a/tasks/keystone_ldap_setup.yml b/tasks/keystone_ldap_setup.yml index 047f8bfd..f24504e6 100644 --- a/tasks/keystone_ldap_setup.yml +++ b/tasks/keystone_ldap_setup.yml @@ -35,10 +35,8 @@ mode: "0640" with_dict: "{{ keystone_ldap }}" notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server # Bug 1547542 - Older versions of the keystone role would deploy a blank # keystone.Default.conf and this will cause errors when adding LDAP-backed @@ -49,7 +47,5 @@ state: absent when: keystone_ldap.Default is not defined notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server diff --git a/tasks/keystone_nginx.yml b/tasks/keystone_nginx.yml index 3e0ab2c6..119bfa78 100644 --- a/tasks/keystone_nginx.yml +++ b/tasks/keystone_nginx.yml @@ -24,8 +24,7 @@ path: /etc/nginx/sites-enabled/default state: absent notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Configure custom nginx log format lineinfile: @@ -33,8 +32,7 @@ dest: "/etc/nginx/nginx.conf" line: "log_format custom '{{ keystone_nginx_access_log_format_combined }} {{ keystone_nginx_access_log_format_extras }}';" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server # Configure app - name: Configure virtual hosts @@ -43,8 +41,7 @@ dest: "/etc/nginx/{{ keystone_nginx_conf_path }}/{{ item }}.conf" with_items: "{{ keystone_wsgi_program_names }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Link to enable virtual hosts file: @@ -54,5 +51,4 @@ with_items: "{{ keystone_wsgi_program_names }}" when: ansible_os_family == "Debian" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server diff --git a/tasks/keystone_post_install.yml b/tasks/keystone_post_install.yml index 5a97c520..53402f18 100644 --- a/tasks/keystone_post_install.yml +++ b/tasks/keystone_post_install.yml @@ -46,10 +46,8 @@ config_type: "json" content: "{{ keystone_policy_user_content | default('{}', true) }}" notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server - name: Copy Keystone Federation SP SSO callback template copy: @@ -61,10 +59,8 @@ when: - keystone_idp != {} notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server - name: Clean up Keystone Federation SP SSO callback template file: @@ -73,7 +69,5 @@ when: - keystone_idp == {} notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes - - Restart web server on first node - - Restart web server on other nodes + - Restart uWSGI + - Restart web server diff --git a/tasks/keystone_ssl_key_create.yml b/tasks/keystone_ssl_key_create.yml index 10122f57..6b48b81c 100644 --- a/tasks/keystone_ssl_key_create.yml +++ b/tasks/keystone_ssl_key_create.yml @@ -29,8 +29,7 @@ -extensions v3_ca creates={{ keystone_ssl_cert }} notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Ensure keystone user owns the self-signed key and certificate file: @@ -42,5 +41,4 @@ - "{{ keystone_ssl_key }}" - "{{ keystone_ssl_cert }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server diff --git a/tasks/keystone_ssl_key_distribute.yml b/tasks/keystone_ssl_key_distribute.yml index a227ceb8..83206f3c 100644 --- a/tasks/keystone_ssl_key_distribute.yml +++ b/tasks/keystone_ssl_key_distribute.yml @@ -21,8 +21,7 @@ group: "{{ keystone_system_group_name }}" mode: "0640" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Distribute self signed ssl cert copy: @@ -32,8 +31,7 @@ group: "{{ keystone_system_group_name }}" mode: "0640" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Ensure keystone user owns the self-signed key and certificate file: @@ -44,5 +42,4 @@ - "{{ keystone_ssl_key }}" - "{{ keystone_ssl_cert }}" notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server diff --git a/tasks/keystone_ssl_user_provided.yml b/tasks/keystone_ssl_user_provided.yml index a7400d84..6e2db52e 100644 --- a/tasks/keystone_ssl_user_provided.yml +++ b/tasks/keystone_ssl_user_provided.yml @@ -22,8 +22,7 @@ mode: "0644" when: keystone_user_ssl_cert is defined notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Drop user provided ssl key copy: @@ -34,8 +33,7 @@ mode: "0640" when: keystone_user_ssl_key is defined notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server - name: Drop user provided ssl CA cert copy: @@ -46,5 +44,4 @@ mode: "0644" when: keystone_user_ssl_ca_cert is defined notify: - - Restart web server on first node - - Restart web server on other nodes + - Restart web server diff --git a/tasks/keystone_uwsgi.yml b/tasks/keystone_uwsgi.yml index fa5bc278..20c4218e 100644 --- a/tasks/keystone_uwsgi.yml +++ b/tasks/keystone_uwsgi.yml @@ -27,8 +27,7 @@ config_type: ini with_items: "{{ keystone_wsgi_program_names }}" notify: - - Restart uWSGI on first node - - Restart uWSGI on other nodes + - Restart uWSGI - include: "keystone_init_{{ ansible_service_mgr }}.yml" vars: diff --git a/tasks/main.yml b/tasks/main.yml index 25019b46..98e4e8e6 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -53,12 +53,14 @@ - name: refresh local facts setup: filter: ansible_local + gather_subset: "!all" tags: - keystone-config - include: keystone_cleanup_old_facts.yml when: - - "ansible_local['keystone'] is defined" + - "('keystone' in ansible_local) or + ('need_db_sync' in ansible_local['openstack_ansible']['keystone'])" tags: - keystone-config @@ -92,19 +94,8 @@ - include: keystone_db_setup.yml when: - - keystone_database_enabled | bool - - inventory_hostname == ansible_play_hosts[0] - - ansible_local['openstack_ansible']['keystone']['need_db_sync'] | bool - tags: - - keystone-config - -- name: Disable the need of a db sync on all nodes - ini_file: - dest: "/etc/ansible/facts.d/openstack_ansible.fact" - section: keystone - option: need_db_sync - value: False - when: dbsync | succeeded + - "keystone_database_enabled | bool" + - "inventory_hostname == ansible_play_hosts[0]" tags: - keystone-config