From 00efa4cfd26de5767ba2a981e6109ad9bc6cd616 Mon Sep 17 00:00:00 2001
From: Jan Marquardt <jm@artfiles.de>
Date: Wed, 10 Jun 2020 11:45:22 +0200
Subject: [PATCH] Added checks for failed cinder services

The task "Perform online data migrations" fails if not all cinder
services are available at the time of execution. Therefore, if
the task fails, we check if all services are up and print a more
meaningful error message.

Change-Id: Ic833be57ad43eb1e471101676fedb07d9ba1e8da
---
 tasks/cinder_db_sync.yml | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/tasks/cinder_db_sync.yml b/tasks/cinder_db_sync.yml
index 70e13ad7..a2b3e889 100644
--- a/tasks/cinder_db_sync.yml
+++ b/tasks/cinder_db_sync.yml
@@ -12,22 +12,42 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 - name: Perform a cinder DB sync
   command: "{{ cinder_bin }}/cinder-manage db sync"
   become: yes
   become_user: "{{ cinder_system_user_name }}"
   changed_when: false
 
-- name: Perform online data migrations
-  command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
+- name: Perform data migrations if necessary and all cinder services are up
+  block:
+
+    - name: Perform online data migrations
+      command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
+      register: data_migrations
+      changed_when: false
+
+  rescue:
+
+    - name: Check if all cinder services are up
+      shell: "{{ cinder_bin }}/cinder-manage service list | /usr/bin/awk '{ if(NR>1) print $5 }' | /bin/egrep -vq ':-)'"
+      register: failed_cinder_services
+      ignore_errors: True
+
+    - name: Fail if any cinder service is down
+      fail:
+        msg: At least one cinder service is down. You need to fix this before re-running the playbook.
+      when: failed_cinder_services.rc != 1
+
+    - name: All cinder services are up, but migration failed nontheless.
+      fail:
+        msg: An unexpected error occured.
+      when: failed_cinder_services.rc == 1
+
   become: yes
   become_user: "{{ cinder_system_user_name }}"
   when:
     - "(cinder_all_software_updated | default('no')) | bool"
     - "ansible_local['openstack_ansible']['cinder']['need_online_data_migrations'] | bool"
-  changed_when: false
-  register: data_migrations
 
 - name: Disable the online migrations requirement
   ini_file: