Move online_data_migrations to post-setup

According to nova rolling upgrade process [1], online_data_migrations
should run once all the services are running the latest version of the
code and were restarted. With that, we should move online migrations
after handlers being flushed, when all services are restarted.

At the same time, nova-status upgrade check must run before services
are restarted to the new version, as service restart might lead to
service breakage if upgrade check fails [2]. It makes no sense to
run upgrade check when upgrade is fully finished.

[1] https://docs.openstack.org/nova/latest/admin/upgrades.html#rolling-upgrade-process
[2] https://docs.openstack.org/nova/latest/cli/nova-status.html#upgrade

Change-Id: Ic681f73a09bb0ac280c227f85c6e79b31fd3429a

tasks/nova_db_post_setup.yml

@@ -25,25 +25,31 @@
   become_user: "{{ _db_nova_system_user_name }}"
   changed_when: false
 
-# The nova-status upgrade check command is typically run after upgrading the
-# controller services to new code, but is also OK to run for a greenfield
-# install to verify everything is setup correctly. This must run after cell
-# mapping setup and online data migrations have run.
-# https://docs.openstack.org/nova/latest/cli/nova-status.html
-- name: Run nova-status upgrade check to validate a healthy configuration
-  command: "{{ _db_nova_bin }}/nova-status upgrade check"
+
+# If it exits with partial updates (exit status 1) it should be called again,
+# even if some updates initially generated errors,
+# because some updates may depend on others having completed. If it exits with status 2,
+# intervention is required to resolve the issue causing remaining updates to fail.
+# It should be considered successfully completed only when the exit status is 0.
+- name: Perform online data migrations
+  command: "{{ _db_nova_bin }}/nova-manage db online_data_migrations"
   become: yes
   become_user: "{{ _db_nova_system_user_name }}"
-  register: nova_status_upgrade_check
-  until: nova_status_upgrade_check is success
-  retries: 8
-  delay: 15
-  # The nova-status upgrade check command has three standard return codes:
-  # 0: all checks were successful
-  # 1: warning: there might be some checks that require investigation, but
-  #    generally will not block an automated install/upgrade; digging into
-  #    warnings is useful for debugging post-install/upgrade issues
-  # 2: at least one check failed and must stop the install/upgrade because
-  #    something was not setup properly
-  failed_when: "nova_status_upgrade_check.rc not in [0, 1]"
-  changed_when: false
+  when:
+    - hostvars[nova_conductor_setup_host]['ansible_local']['openstack_ansible']['nova']['need_online_data_migrations'] | bool
+  retries: 5
+  delay: 3
+  until: "data_migrations.rc in [0, 2]"
+  register: data_migrations
+
+- name: Disable the online migrations requirement
+  delegate_to: "{{ item }}"
+  ini_file:
+    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
+    section: nova
+    option: need_online_data_migrations
+    value: False
+  with_items: "{{ groups[nova_services['nova-conductor']['group']] }}"
+  when:
+    - data_migrations is not skipped
+    - data_migrations is succeeded

tasks/nova_db_setup.yml

@@ -35,26 +35,6 @@
   become_user: "{{ nova_system_user_name }}"
   changed_when: false
 
-- name: Perform online data migrations
-  command: "{{ nova_bin }}/nova-manage db online_data_migrations"
-  become: yes
-  become_user: "{{ nova_system_user_name }}"
-  when:
-    - "(nova_all_software_updated | default('no')) | bool"
-    - "ansible_local['openstack_ansible']['nova']['need_online_data_migrations'] | bool"
-  changed_when: false
-  register: data_migrations
-
-- name: Disable the online migrations requirement
-  ini_file:
-    dest: "/etc/ansible/facts.d/openstack_ansible.fact"
-    section: nova
-    option: need_online_data_migrations
-    value: False
-  when:
-    - data_migrations is not skipped
-    - data_migrations is succeeded
-
 # We need to check for existance of the cell, since nova-manage cell_v2 create_cell
 # might be not idempotent due to the bug https://bugs.launchpad.net/nova/+bug/1923899
 - name: Get UUID of new Nova Cell
@@ -85,3 +65,28 @@
   failed_when: "nova_cell1_create.rc not in [0, 2]"
   changed_when: "nova_cell1_create.rc == 0"
   when: "_cell_uuid.rc == 1"
+
+# The nova-status upgrade check command is typically run after upgrading the
+# controller services to new code, but is also OK to run for a greenfield
+# install to verify everything is setup correctly. This must run after cell
+# mapping setup and before actual service restart.
+# https://docs.openstack.org/nova/latest/cli/nova-status.html
+- name: Run nova-status upgrade check to validate a healthy configuration
+  command: "{{ nova_bin }}/nova-status upgrade check"
+  become: yes
+  become_user: "{{ nova_system_user_name }}"
+  register: nova_status_upgrade_check
+  until: nova_status_upgrade_check is success
+  retries: 8
+  delay: 15
+  # The nova-status upgrade check command has three standard return codes:
+  # 0: all checks were successful
+  # 1: warning: there might be some checks that require investigation, but
+  #    generally will not block an automated install/upgrade; digging into
+  #    warnings is useful for debugging post-install/upgrade issues
+  # 2: at least one check failed and must stop the install/upgrade because
+  #    something was not setup properly
+  failed_when: "nova_status_upgrade_check.rc not in [0, 1]"
+  changed_when: false
+  when:
+    - "ansible_local['openstack_ansible']['nova']['need_online_data_migrations'] | bool"