From 7fc1497ebe293444be13854009e5fcd3b2b64449 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Fri, 30 Jun 2017 19:35:01 +0100 Subject: [PATCH] Implement data migrations for rolling upgrades In order to cater for artifact-based installed, and rolling upgrades, this patch implements a set of local facts to inform the online migrations task. The 'nova_all_software_updated' variable will be set by the playbook on each run to ensure that the online migrations only happen once all venvs are homogenous. This ensures that the playbook can be executed in a serialised fashion and the data will not be corrupted. The ``upgrade_levels`` setting for ``compute`` is set to ``auto`` to ensure that a mixed RPC version deployment can operate properly when doing a rolling upgrade as suggested by [1]. Additional changes are made to improve the role's ability to be executed using serialised playbooks. Finally, the nova-manage command references to the config file location have been removed as they refer to the default location. [1] https://docs.openstack.org/developer/nova/upgrade.html Change-Id: I08e5a7f0ce526b11aa52c35ee29c458954a5f22d --- handlers/main.yml | 2 +- ...nova-data-migrations-f6c2bc0f9e7b1908.yaml | 9 ++++ tasks/main.yml | 36 +++++++++++--- tasks/nova_db_post_setup.yml | 28 ++--------- tasks/nova_db_setup.yml | 49 +++++++++---------- tasks/nova_install.yml | 19 +++++++ templates/nova.conf.j2 | 3 ++ 7 files changed, 87 insertions(+), 59 deletions(-) create mode 100644 releasenotes/notes/nova-data-migrations-f6c2bc0f9e7b1908.yaml diff --git a/handlers/main.yml b/handlers/main.yml index 6bd2af2c..12c9a35a 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -86,4 +86,4 @@ retries: 5 delay: 2 when: - - inventory_hostname in groups['nova_api_placement'] + - "'nova_api_placement' in group_names" diff --git a/releasenotes/notes/nova-data-migrations-f6c2bc0f9e7b1908.yaml b/releasenotes/notes/nova-data-migrations-f6c2bc0f9e7b1908.yaml new file mode 100644 index 00000000..0724c27e --- /dev/null +++ b/releasenotes/notes/nova-data-migrations-f6c2bc0f9e7b1908.yaml @@ -0,0 +1,9 @@ +--- +features: + - The ``os_nova`` role now provides for doing + online data migrations once the db sync has + been completed. The data migrations will not + be executed until the boolean variable + ``nova_all_software_updated`` is true. This + variable will need to be set by the playbook + consuming the role. diff --git a/tasks/main.yml b/tasks/main.yml index 6b653792..27d0a7e9 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -49,13 +49,22 @@ tags: - nova-install +- name: refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - nova-config + - include: nova_post_install.yml tags: - nova-config - include: nova_db_setup.yml + static: no when: - - inventory_hostname == groups['nova_api_os_compute'][0] + - "'nova_conductor' in group_names" + - "inventory_hostname == ansible_play_hosts[0]" tags: - nova-config @@ -64,29 +73,44 @@ - nova-config - include: nova_service_setup.yml + static: no when: - - inventory_hostname == groups['nova_api_os_compute'][0] + - "'nova_conductor' in group_names" + - "inventory_hostname == ansible_play_hosts[0]" tags: - nova-config - include: nova_placement.yml + static: no when: - - nova_placement_service_enabled | bool - - inventory_hostname in groups['nova_api_placement'] + - "nova_placement_service_enabled | bool" + - "'nova_api_placement' in group_names" tags: - nova-placement - include: nova_compute.yml + static: no when: - - inventory_hostname in groups['nova_compute'] + - "'nova_compute' in group_names" tags: - nova-compute - name: Flush handlers meta: flush_handlers +# We have to delegate this back to the conductor +# because the compute hosts do not have access to +# the database connection string and therefore +# cannot run nova-manage. +# Also, we delegate this to a random host in the +# conductor group in order to spread the load of +# multiple forks across the group members. - include: nova_db_post_setup.yml + delegate_to: "{{ conductor_host }}" + with_random_choice: "{{ groups['nova_conductor'] }}" + loop_control: + loop_var: conductor_host when: - - inventory_hostname == groups['nova_api_os_compute'][0] + - "'nova_compute' in group_names" tags: - nova-config diff --git a/tasks/nova_db_post_setup.yml b/tasks/nova_db_post_setup.yml index 22ae9792..15e7a772 100644 --- a/tasks/nova_db_post_setup.yml +++ b/tasks/nova_db_post_setup.yml @@ -19,40 +19,18 @@ become: yes become_user: "{{ nova_system_user_name }}" changed_when: false - tags: - - nova-db-setup - - nova-setup - - nova-command-bin # When upgrading we need to map existing instances to the new cell1 # To do this we need the cell UUID. - name: Get UUID of new Nova Cell - shell: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf cell_v2 list_cells | awk '/{{ nova_cell1_name }}/ {print $4}'" + shell: "{{ nova_bin }}/nova-manage cell_v2 list_cells | grep ' {{ nova_cell1_name }} '" become: yes become_user: "{{ nova_system_user_name }}" - when: "nova_cell1_create.rc == 0" register: cell1_uuid - tags: - - nova-db-setup - - nova-setup - - nova-command-bin + changed_when: false - name: Map instances to new Cell1 - command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf cell_v2 map_instances --cell_uuid {{ cell1_uuid.stdout }}" - become: yes - become_user: "{{ nova_system_user_name }}" - when: "nova_cell1_create.rc == 0" - tags: - - nova-db-setup - - nova-setup - - nova-command-bin - -- name: Perform Nova online data migrations - command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf db online_data_migrations" + command: "{{ nova_bin }}/nova-manage cell_v2 map_instances --cell_uuid {{ cell1_uuid['stdout'].split()[3] }}" become: yes become_user: "{{ nova_system_user_name }}" changed_when: false - tags: - - nova-db-setup - - nova-setup - - nova-command-bin diff --git a/tasks/nova_db_setup.yml b/tasks/nova_db_setup.yml index 662c5b3c..cc19ac73 100644 --- a/tasks/nova_db_setup.yml +++ b/tasks/nova_db_setup.yml @@ -19,32 +19,20 @@ become_user: "{{ nova_system_user_name }}" register: nova_api_db_version changed_when: false - tags: - - nova-db-setup - - nova-setup - - nova-command-bin - name: Perform a Nova API DB sync - command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf api_db sync" + command: "{{ nova_bin }}/nova-manage api_db sync" become: yes become_user: "{{ nova_system_user_name }}" changed_when: false when: - nova_api_db_version.stdout == "0" - tags: - - nova-db-setup - - nova-setup - - nova-command-bin - name: Perform cell_v2 map cell0 command: "{{ nova_bin }}/nova-manage cell_v2 map_cell0 --database_connection mysql+pymysql://{{ nova_api_galera_user }}:{{ nova_api_container_mysql_password }}@{{ nova_api_galera_address }}/{{ nova_cell0_database }}?charset=utf8" become: yes become_user: "{{ nova_system_user_name }}" changed_when: false - tags: - - nova-db-setup - - nova-setup - - nova-command-bin - name: Perform cell_v2 initial cell setup command: "{{ nova_bin }}/nova-manage cell_v2 create_cell --name {{ nova_cell1_name }} --database_connection mysql+pymysql://{{ nova_galera_user }}:{{ nova_container_mysql_password }}@{{ nova_galera_address }}/{{ nova_galera_database }}?charset=utf8 --transport-url rabbit://{% for host in nova_rabbitmq_servers.split(',') %}{{ nova_rabbitmq_userid }}:{{ nova_rabbitmq_password }}@{{ host }}:{{ nova_rabbitmq_port }}{% if not loop.last %},{% else %}/{{ nova_rabbitmq_vhost }}{% endif %}{% endfor %}" @@ -53,27 +41,34 @@ register: nova_cell1_create failed_when: "nova_cell1_create.rc not in [0, 2]" changed_when: "nova_cell1_create.rc == 0" - tags: - - nova-db-setup - - nova-setup - - nova-command-bin - name: Perform a Nova API DB sync - command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf api_db sync" + command: "{{ nova_bin }}/nova-manage api_db sync" become: yes become_user: "{{ nova_system_user_name }}" changed_when: false - tags: - - nova-db-setup - - nova-setup - - nova-command-bin - name: Perform a Nova DB sync - command: "{{ nova_bin }}/nova-manage --config-file /etc/nova/nova.conf db sync" + command: "{{ nova_bin }}/nova-manage db sync" become: yes become_user: "{{ nova_system_user_name }}" changed_when: false - tags: - - nova-db-setup - - nova-setup - - nova-command-bin + +- name: Perform online data migrations + command: "{{ nova_bin }}/nova-manage db online_data_migrations" + become: yes + become_user: "{{ nova_system_user_name }}" + when: + - "(nova_all_software_updated | default('no')) | bool" + - "ansible_local['openstack_ansible']['nova']['need_online_data_migrations'] | bool" + changed_when: false + register: data_migrations + +- name: Disable the online migrations requirement + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: nova + option: need_online_data_migrations + value: False + when: + - data_migrations | succeeded diff --git a/tasks/nova_install.yml b/tasks/nova_install.yml index 38d1c7d2..1a75f9bf 100644 --- a/tasks/nova_install.yml +++ b/tasks/nova_install.yml @@ -165,6 +165,25 @@ - nova-novnc-console - nova-spice-console +- name: Initialise the upgrade facts + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: nova + option: "{{ item }}" + value: True + with_items: + - "need_service_restart" + - "need_online_data_migrations" + when: + - (nova_get_venv | changed) or + (nova_venv_dir | changed) or + (install_packages | changed) or + (ansible_local is not defined) or + ('openstack_ansible' not in ansible_local) or + ('nova' not in ansible_local['openstack_ansible']) or + ('need_online_data_migrations' not in ansible_local['openstack_ansible']['nova']) or + ('need_service_restart' not in ansible_local['openstack_ansible']['nova']) + - name: Record the venv tag deployed ini_file: dest: "/etc/ansible/facts.d/openstack_ansible.fact" diff --git a/templates/nova.conf.j2 b/templates/nova.conf.j2 index 2db262c9..9cd77e6f 100644 --- a/templates/nova.conf.j2 +++ b/templates/nova.conf.j2 @@ -324,3 +324,6 @@ metadata_items = {{ nova_quota_metadata_items }} ram = {{ nova_quota_ram }} server_group_members = {{ nova_quota_server_group_members }} server_groups = {{ nova_quota_server_groups }} + +[upgrade_levels] +compute=auto