Block site_update if there is no host in MaaS/Drydock

Added the feature in airflow to verify that MaaS list
of BM hosts is not empty for shipyard update_site action.
If the MaaS Machine list is empty, and the
continue-on-fail parameter is not set to true (the default
value is false), it will fail the shipyard steps to
parepare and re-deploy the missing nodes in MaaS through
DD.
Caveat: this US did not have the requirement to compare
the list of nodes in MaaS with the expected site Design.
It simply checks for empty node list, and decide based
on that.

Change-Id: I5ba4a107fe2ae43728e5941570b6c88a436d7b12

src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py

@@ -174,7 +174,8 @@ class CommonStepFactory(object):
             on_failure_callback=step_failure_handler,
             dag=self.dag)
 
-    def get_drydock_build(self, task_id=dn.DRYDOCK_BUILD_DAG_NAME):
+    def get_drydock_build(self, task_id=dn.DRYDOCK_BUILD_DAG_NAME,
+                          verify_nodes_exist=False):
         """Generate the drydock build step
 
         Drydock build does the hardware provisioning.
@@ -183,7 +184,8 @@ class CommonStepFactory(object):
             subdag=deploy_site_drydock(
                 self.parent_dag_name,
                 task_id,
-                args=self.default_args),
+                args=self.default_args,
+                verify_nodes_exist=verify_nodes_exist),
             task_id=task_id,
             on_failure_callback=step_failure_handler,
             dag=self.dag)

src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py

@@ -18,6 +18,7 @@ try:
     from airflow.operators import DrydockNodesOperator
     from airflow.operators import DrydockPrepareSiteOperator
     from airflow.operators import DrydockVerifySiteOperator
+    from airflow.operators import DrydockVerifyNodesExistOperator
     from config_path import config_path
 except ImportError:
     from shipyard_airflow.plugins.drydock_nodes import \
@@ -26,10 +27,13 @@ except ImportError:
         DrydockPrepareSiteOperator
     from shipyard_airflow.plugins.drydock_verify_site import \
         DrydockVerifySiteOperator
+    from shipyard_airflow.plugins.drydock_verify_nodes import \
+        DrydockVerifyNodesExistOperator
     from shipyard_airflow.dags.config_path import config_path
 
 
-def deploy_site_drydock(parent_dag_name, child_dag_name, args):
+def deploy_site_drydock(parent_dag_name, child_dag_name, args,
+                        verify_nodes_exist=False):
     '''
     DryDock Subdag
     '''
@@ -37,6 +41,13 @@ def deploy_site_drydock(parent_dag_name, child_dag_name, args):
         '{}.{}'.format(parent_dag_name, child_dag_name),
         default_args=args)
 
+    if verify_nodes_exist:
+        drydock_verify_nodes_exist = DrydockVerifyNodesExistOperator(
+            task_id='verify_nodes_exist',
+            shipyard_conf=config_path,
+            main_dag_name=parent_dag_name,
+            dag=dag)
+
     drydock_verify_site = DrydockVerifySiteOperator(
         task_id='verify_site',
         shipyard_conf=config_path,
@@ -57,6 +68,10 @@ def deploy_site_drydock(parent_dag_name, child_dag_name, args):
 
     # Define dependencies
     drydock_prepare_site.set_upstream(drydock_verify_site)
-    drydock_nodes.set_upstream(drydock_prepare_site)
+    if verify_nodes_exist:
+        drydock_verify_nodes_exist.set_upstream(drydock_prepare_site)
+        drydock_nodes.set_upstream(drydock_verify_nodes_exist)
+    else:
+        drydock_nodes.set_upstream(drydock_prepare_site)
 
     return dag

src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py

@@ -58,7 +58,7 @@ preflight = step_factory.get_preflight()
 get_rendered_doc = step_factory.get_get_rendered_doc()
 deployment_configuration = step_factory.get_deployment_configuration()
 validate_site_design = step_factory.get_validate_site_design()
-drydock_build = step_factory.get_drydock_build()
+drydock_build = step_factory.get_drydock_build(verify_nodes_exist=True)
 armada_build = step_factory.get_armada_build()
 decide_airflow_upgrade = step_factory.get_decide_airflow_upgrade()
 upgrade_airflow = step_factory.get_upgrade_airflow()

src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py

@@ -368,6 +368,22 @@ class DrydockBaseOperator(UcpBaseOperator):
                         "Unable to retrieve subtask info!"
                     )
 
+    def get_nodes(self):
+        """
+        Get the list of all the build data record for all nodes(hostname)
+        in raw dictionary format.
+
+        Raises DrydockClientUseFailureException if the client raises an
+        exception
+        See:
+        https://att-comdev-drydock.readthedocs.io/en/latest/API.html
+        """
+        try:
+            return self.drydock_client.get_nodes()
+        except errors.ClientError as client_error:
+            LOG.error("Drydock client failed to get nodes from Drydock.")
+            raise DrydockClientUseFailureException(client_error)
+
     def get_successes_for_task(self, task_id, extend_success=True):
         """Discover the successful nodes based on the current task id.
 

src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_nodes.py

@@ -0,0 +1,60 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from airflow.exceptions import AirflowException
+from airflow.plugins_manager import AirflowPlugin
+
+try:
+    from drydock_base_operator import DrydockBaseOperator
+except ImportError:
+    from shipyard_airflow.plugins.drydock_base_operator import \
+        DrydockBaseOperator
+
+LOG = logging.getLogger(__name__)
+
+
+class DrydockVerifyNodesExistOperator(DrydockBaseOperator):
+
+    """Drydock Verify nodes exist Operator
+
+    This operator will trigger drydock to verify node for
+    site update
+
+    """
+
+    def do_execute(self):
+
+        LOG.info('verify_nodes_exit was invoked.')
+        node_list = self.get_nodes()
+        continue_on_fail = self.action_info['parameters'].get(
+            'continue-on-fail', 'false')
+        LOG.debug('node list is : {}'.format(node_list))
+        LOG.debug('continue on fail is: {}'.format(continue_on_fail))
+
+        if not node_list and str(continue_on_fail).lower() != 'true':
+            msg = 'No nodes were found in MaaS, ' \
+                  'and continue_on_fail is {} ' \
+                  '-> Fail Drydock prepare and ' \
+                  'deply nodes.'.format(continue_on_fail)
+            LOG.error(msg)
+            raise AirflowException(msg)
+
+
+class DrydockVerifyNodesExistOperatorPlugin(AirflowPlugin):
+
+    """Creates DrydockVerifyNodesExistOperatorPlugin in Airflow."""
+
+    name = 'drydock_verify_nodes_exist_operator'
+    operators = [DrydockVerifyNodesExistOperator]