Added checks for failed cinder services

The task "Perform online data migrations" fails if not all cinder
services are available at the time of execution. Therefore, if
the task fails, we check if all services are up and print a more
meaningful error message.

Change-Id: Ic833be57ad43eb1e471101676fedb07d9ba1e8da

tasks/cinder_db_sync.yml

@@ -12,22 +12,42 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 - name: Perform a cinder DB sync
   command: "{{ cinder_bin }}/cinder-manage db sync"
   become: yes
   become_user: "{{ cinder_system_user_name }}"
   changed_when: false
 
-- name: Perform online data migrations
-  command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
+- name: Perform data migrations if necessary and all cinder services are up
+  block:
+
+    - name: Perform online data migrations
+      command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
+      register: data_migrations
+      changed_when: false
+
+  rescue:
+
+    - name: Check if all cinder services are up
+      shell: "{{ cinder_bin }}/cinder-manage service list | /usr/bin/awk '{ if(NR>1) print $5 }' | /bin/egrep -vq ':-)'"
+      register: failed_cinder_services
+      ignore_errors: True
+
+    - name: Fail if any cinder service is down
+      fail:
+        msg: At least one cinder service is down. You need to fix this before re-running the playbook.
+      when: failed_cinder_services.rc != 1
+
+    - name: All cinder services are up, but migration failed nontheless.
+      fail:
+        msg: An unexpected error occured.
+      when: failed_cinder_services.rc == 1
+
   become: yes
   become_user: "{{ cinder_system_user_name }}"
   when:
     - "(cinder_all_software_updated | default('no')) | bool"
     - "ansible_local['openstack_ansible']['cinder']['need_online_data_migrations'] | bool"
-  changed_when: false
-  register: data_migrations
 
 - name: Disable the online migrations requirement
   ini_file: