Use TripleO Heat Merge to manage the stack

Generate the YAML file describing Tuskar compute nodes and invoke Heat
Merge. This review adds the master branch of tripleo-heat-templates as
a requirement.

At the moment this patch hardcodes the parameters which define the
different Overcloud Roles. On a create over the Overcloud it creates one
controller and one compute Overcloud role. When the Overcloud is updated
it created a second compute node.

This can be manually tested with these commands.

Create the overcloud:
curl -H "Content-Type:application/json"\
 -XPOST http://localhost:8585/v1/overclouds/ -d '{}'

Update the overcloud:
curl -H "Content-Type:application/json"\
 -XPUT http://localhost:8585/v1/overclouds/1 -d '{}'

Delete the overcloud:
curl -H "Content-Type:application/json"\
 -XDELETE http://localhost:8585/v1/overclouds/1 -d '{}'

Change-Id: I578b4e9f238590ea245b827bc75d252568d194fe

requirements.txt

@@ -20,3 +20,5 @@ WSME>=0.5b6
 python-novaclient>=2.15.0
 PyYAML>=3.1.0
 python-heatclient>=0.2.3
+
+-e git+http://git.openstack.org/cgit/openstack/tripleo-heat-templates#egg=tripleo_heat_templates-master

setup.cfg

@@ -34,7 +34,7 @@ build-dir = doc/build
 source-dir = doc/source
 
 [egg_info]
-tag_build = 
+tag_build =
 tag_date = 0
 tag_svn_revision = 0
 

tuskar/api/controllers/v1/overcloud.py

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -14,19 +12,70 @@
 
 import logging
 import pecan
-from pecan import rest
 import wsme
+
+from pecan import rest
 from wsmeext import pecan as wsme_pecan
 
 from tuskar.api.controllers.v1 import models
-
+from tuskar.common import exception
+from tuskar.heat.client import HeatClient
+import tuskar.heat.template_tools as template_tools
 
 LOG = logging.getLogger(__name__)
 
 
+# FIXME(lsmola) mocked params for POC, remove later by real ones
+POC_PARAMS = {'controller': 1, 'compute': 1}
+POC_PARAMS_UPDATE = {'controller': 1, 'compute': 2}
+
+
+def process_stack(params, create=False):
+    """Helper function for processing the stack. Given a params dict containing
+    the Overcloud Roles and initialization parameters create or update the
+    stack.
+
+    :param params: Dictionary of initialization params and overcloud roles for
+                   heat template and initialization of stack/
+    :type  params: dict
+
+    :param create: A flag to designate if we are creating or updating the stack
+    :type create: bool
+    """
+
+    overcloud = template_tools.merge_templates(params)
+    heat_client = HeatClient()
+
+    stack_exists = heat_client.exists_stack()
+    if not heat_client.validate_template(overcloud):
+        raise exception.InvalidHeatTemplate()
+
+    if stack_exists and create:
+        raise exception.StackAlreadyCreated()
+
+    elif not stack_exists and not create:
+        raise exception.StackNotFound()
+
+    res = heat_client.create_stack(overcloud, params)
+
+    if not res:
+        if create:
+            raise exception.HeatTemplateCreateFailed()
+
+        raise exception.HeatTemplateUpdateFailed()
+
+
 class OvercloudsController(rest.RestController):
     """REST controller for the Overcloud class."""
 
+    _custom_actions = {'template_get': ['GET']}
+
+    # FIXME(lsmola) this is for debugging purposes only, remove before I3
+    @pecan.expose()
+    def template_get(self):
+        overcloud = template_tools.merge_templates(POC_PARAMS)
+        return overcloud
+
     @wsme.validate(models.Overcloud)
     @wsme_pecan.wsexpose(models.Overcloud,
                          body=models.Overcloud,
@@ -55,6 +104,14 @@ class OvercloudsController(rest.RestController):
         saved_overcloud =\
             models.Overcloud.from_db_model(result)
 
+        # FIXME(lsmola) This is just POC of creating a stack
+        # this has to be done properly with proper Work-flow abstraction of:
+        # step one- build template and start stack-create
+        # step 2- put the right stack_id to the overcloud
+        # step 3- initialize the stack
+        # step 4- set the correct overcloud status
+        process_stack(POC_PARAMS, create=True)
+
         return saved_overcloud
 
     @wsme.validate(models.Overcloud)
@@ -90,6 +147,12 @@ class OvercloudsController(rest.RestController):
 
         updated = models.Overcloud.from_db_model(result)
 
+        # FIXME(lsmola) This is just POC of updating a stack
+        # this probably should also have workflow
+        # step one- build template and stack-update
+        # step 2- set the correct overcloud status
+        process_stack(POC_PARAMS_UPDATE)
+
         return updated
 
     @wsme_pecan.wsexpose(None, int, status_code=204)
@@ -103,9 +166,23 @@ class OvercloudsController(rest.RestController):
                  is no overcloud with the given ID
         """
 
+        # FIXME(lsmola) this should always try to delete both overcloud
+        # and stack. So it requires some exception catch over below.
         LOG.debug('Deleting overcloud with ID: %s' % overcloud_id)
         pecan.request.dbapi.delete_overcloud_by_id(overcloud_id)
 
+        heat_client = HeatClient()
+        if not heat_client.exists_stack():
+            # If the stack doesn't exist, we have nothing else to do here.
+            return
+
+        result = heat_client.delete_stack()
+
+        if not result:
+            raise wsme.exc.ClientSideError(_(
+                "Failed to delete the Heat overcloud."
+            ))
+
     @wsme_pecan.wsexpose(models.Overcloud, int)
     def get_one(self, overcloud_id):
         """Returns a specific overcloud.

tuskar/common/exception.py

@@ -170,3 +170,23 @@ class DuplicateAttribute(DuplicateEntry):
 
 class ConfigNotFound(TuskarException):
     message = _("Could not find config at %(path)s")
+
+
+class InvalidHeatTemplate(TuskarException):
+    message = _("Validation of the Heat Template failed.")
+
+
+class StackNotFound(NotFound):
+    message = _("The Stack for this Overcloud can't be found.")
+
+
+class StackAlreadyCreated(DuplicateEntry):
+    message = _("The Stack for this Overcloud already exists.")
+
+
+class HeatTemplateUpdateFailed(TuskarException):
+    message = _("The Heat template failed to update.")
+
+
+class HeatTemplateCreateFailed(TuskarException):
+    message = _("The Heat template failed to create.")

tuskar/heat/client.py

@@ -80,15 +80,16 @@ class HeatClient(object):
         try:
             keystone = ksclient.Client(**CONF.heat_keystone)
             endpoint = keystone.service_catalog.url_for(
-                    service_type=CONF.heat['service_type'],
+                service_type=CONF.heat['service_type'],
-                    endpoint_type=CONF.heat['endpoint_type'])
+                endpoint_type=CONF.heat['endpoint_type']
+            )
             self.connection = heatclient(
                 endpoint=endpoint,
                 token=keystone.auth_token,
                 username=CONF.heat_keystone['username'],
                 password=CONF.heat_keystone['password'])
-        except Exception as e:
+        except Exception:
-            LOG.exception(e)
+            LOG.exception("An error occurred initialising the HeatClient")
             self.connection = None
 
     def validate_template(self, template_body):
@@ -96,8 +97,8 @@ class HeatClient(object):
         try:
             self.connection.stacks.validate(template=template_body)
             return True
-        except Exception as e:
+        except Exception:
-            LOG.exception(e)
+            LOG.exception("Validation of the Heat template failed.")
             return False
 
     def get_stack(self, name=None):
@@ -110,8 +111,8 @@ class HeatClient(object):
     def get_template(self):
         """Get JSON representation of the Heat overcloud template."""
         return self.connection.stacks.template(
-                stack_id=CONF.heat['stack_name']
+            stack_id=CONF.heat['stack_name']
-                )
+        )
 
     def update_stack(self, template_body, params):
         """Update the Heat overcloud stack."""
@@ -120,8 +121,17 @@ class HeatClient(object):
                                           template=template_body,
                                           parameters=params)
             return True
-        except Exception as e:
+        except Exception:
-            LOG.exception(e)
+            LOG.exception("An error occurred updating the stack.")
+            return False
+
+    def delete_stack(self):
+        """Delete the Heat overcloud stack."""
+        try:
+            self.connection.stacks.delete(stack_id=CONF.heat['stack_name'])
+            return True
+        except Exception:
+            LOG.exception("An error occurred deleting the stack.")
             return False
 
     def create_stack(self, template_body, params):
@@ -131,8 +141,8 @@ class HeatClient(object):
                                           template=template_body,
                                           parameters=params)
             return True
-        except Exception as e:
+        except Exception:
-            LOG.exception(e)
+            LOG.exception("An error occurred creating the stack.")
             return False
 
     def exists_stack(self, name=None):

tuskar/heat/template_tools.py

@@ -0,0 +1,78 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Utilities for using merge.py to generate overcloud.yaml to hand over to Heat.
+Translates Tuskar resources into the overcloud heat template, using merge.py
+from upstream tripleo-heat-templates.
+"""
+
+import os
+
+from oslo.config import cfg
+from tripleo_heat_merge import merge
+
+
+# The name of the compute Overcloud role - defined for special case handling
+OVERCLOUD_COMPUTE_ROLE = 'compute'
+
+
+def generate_scaling_params(overcloud_roles):
+    """Given a dictionary containing a key value mapping of Overcloud Role name
+    to a count of the nodes return the scaling parameters to be used by
+    tripleo_heat_merge
+
+    :param overcloud_roles: Dictionary with role names and a count of the nodes
+    :type  overcloud_roles: dict
+
+    :return: scaling parameters dict
+    :rtype:  dict
+    """
+
+    scaling = {}
+
+    for overcloud_role, count in overcloud_roles.items():
+        overcloud_role = overcloud_role.lower()
+
+        if overcloud_role == OVERCLOUD_COMPUTE_ROLE:
+            scaling = dict(scaling.items() +
+                merge.parse_scaling(["NovaCompute=%s" % (count)]).items())
+
+    return scaling
+
+
+def _join_template_path(file_name):
+    return os.path.abspath(
+        os.path.join(os.path.dirname(cfg.CONF.tht_local_dir), file_name)
+    )
+
+
+def merge_templates(overcloud_roles):
+    """Merge the Overcloud Roles with overcloud.yaml using merge from
+    tripleo_heat_merge
+
+    See tripleo-heat-templates for further details.
+    """
+
+    # TODO(dmatthews): Add exception handling to catch merge errors
+
+    scale_params = generate_scaling_params(overcloud_roles)
+    overcloud_src_path = _join_template_path("overcloud-source.yaml")
+    ssl_src_path = _join_template_path("ssl-source.yaml")
+    swift_src_path = _join_template_path("swift-source.yaml")
+
+    template = merge.merge(
+        [overcloud_src_path, ssl_src_path, swift_src_path], None, None,
+        included_template_dir=cfg.CONF.tht_local_dir, scaling=scale_params
+    )
+
+    return template

tuskar/tests/api/controllers/v1/test_overcloud.py

@@ -12,11 +12,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import mock
 import os
 
-import mock
 from pecan.testing import load_test_app
 
+from tuskar.api.controllers.v1 import overcloud
+from tuskar.common import exception
 from tuskar.db.sqlalchemy import models as db_models
 from tuskar.tests import base
 
@@ -69,13 +71,85 @@ class OvercloudTests(base.TestCase):
 
         mock_db_get.assert_called_once_with(12345)
 
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': True,
+            'exists_stack.return_value': False,
+            'create_stack.return_value': True,
+        })
+    )
+    def test_create_stack(self, mock_heat_client, mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test
+        response = overcloud.process_stack({}, create=True)
+
+        # Verify
+        self.assertEqual(response, None)
+
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': True,
+            'exists_stack.return_value': False,
+            'create_stack.return_value': False,
+        })
+    )
+    def test_create_stack_heat_exception(self, mock_heat_client,
+            mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test and Verify
+        self.assertRaises(
+            exception.HeatTemplateCreateFailed,
+            overcloud.process_stack, {}, True)
+
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': True,
+            'exists_stack.return_value': True,
+            'create_stack.return_value': True,
+        })
+    )
+    def test_create_stack_existing_exception(self, mock_heat_client,
+            mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test and Verify
+        self.assertRaises(
+            exception.StackAlreadyCreated, overcloud.process_stack, {}, True)
+
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': False,
+            'exists_stack.return_value': False,
+            'create_stack.return_value': True,
+        })
+    )
+    def test_create_stack_not_valid_exception(self, mock_heat_client,
+            mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test and Verify
+        self.assertRaises(
+            exception.InvalidHeatTemplate, overcloud.process_stack, {}, True)
+
+    @mock.patch('tuskar.api.controllers.v1.overcloud.process_stack')
     @mock.patch('tuskar.db.sqlalchemy.api.Connection.create_overcloud')
-    def test_post(self, mock_db_create):
+    def test_post(self, mock_db_create, mock_process_stack):
         # Setup
         create_me = {'name': 'new'}
 
         fake_created = db_models.Overcloud(name='created')
         mock_db_create.return_value = fake_created
+        mock_process_stack.return_value = None
 
         # Test
         response = self.app.post_json(URL_OVERCLOUDS, params=create_me)
@@ -91,8 +165,78 @@ class OvercloudTests(base.TestCase):
                                    db_models.Overcloud))
         self.assertEqual(db_create_model.name, create_me['name'])
 
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': True,
+            'exists_stack.return_value': True,
+            'create_stack.return_value': True,
+        })
+    )
+    def test_update_stack(self, mock_heat_client, mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test
+        response = overcloud.process_stack({})
+
+        # Verify
+        self.assertEqual(response, None)
+
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': True,
+            'exists_stack.return_value': True,
+            'create_stack.return_value': False,
+        })
+    )
+    def test_update_stack_heat_exception(self, mock_heat_client,
+            mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test and Verify
+        self.assertRaises(
+            exception.HeatTemplateUpdateFailed, overcloud.process_stack, {})
+
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': True,
+            'exists_stack.return_value': False,
+            'create_stack.return_value': True,
+        })
+    )
+    def test_update_stack_not_existing_exception(self, mock_heat_client,
+            mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test and Verify
+        self.assertRaises(
+            exception.StackNotFound, overcloud.process_stack, {})
+
+    @mock.patch('tuskar.heat.template_tools.merge_templates')
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'validate_template.return_value': False,
+            'exists_stack.return_value': True,
+            'create_stack.return_value': True,
+        })
+    )
+    def test_update_stack_not_valid_exception(self, mock_heat_client,
+            mock_heat_merge_templates):
+        # Setup
+        mock_heat_merge_templates.return_value = None
+
+        # Test and Verify
+        self.assertRaises(
+            exception.InvalidHeatTemplate, overcloud.process_stack, {})
+
+    @mock.patch('tuskar.api.controllers.v1.overcloud.process_stack')
     @mock.patch('tuskar.db.sqlalchemy.api.Connection.update_overcloud')
-    def test_put(self, mock_db_update):
+    def test_put(self, mock_db_update, mock_process_stack):
         # Setup
         changes = {'name': 'updated'}
 
@@ -100,6 +244,7 @@ class OvercloudTests(base.TestCase):
                                            attributes=[],
                                            counts=[])
         mock_db_update.return_value = fake_updated
+        mock_process_stack.return_value = None
 
         # Test
         url = URL_OVERCLOUDS + '/' + '12345'
@@ -119,7 +264,12 @@ class OvercloudTests(base.TestCase):
 
     @mock.patch('tuskar.db.sqlalchemy.api.'
                 'Connection.delete_overcloud_by_id')
-    def test_delete(self, mock_db_delete):
+    @mock.patch(
+        'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{
+            'delete_stack.return_value': True,
+        })
+    )
+    def test_delete(self, mock_heat_client, mock_db_delete):
         # Test
         url = URL_OVERCLOUDS + '/' + '12345'
         response = self.app.delete(url)

tuskar/tests/conf_fixture.py

@@ -42,5 +42,7 @@ class ConfFixture(fixtures.Fixture):
         self.conf.set_default('sqlite_synchronous', False)
         self.conf.set_default('use_ipv6', True)
         self.conf.set_default('verbose', True)
+        self.conf.set_default('tht_local_dir',
+                              '/etc/tuskar/tripleo-heat-templates/')
         config.parse_args([], default_config_files=[])
         self.addCleanup(self.conf.reset)

tuskar/tests/heat/test_template_tools.py

@@ -0,0 +1,53 @@
+# -*- encoding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import unittest
+
+from tuskar.heat import template_tools
+
+
+class TemplateToolsTests(unittest.TestCase):
+
+    @mock.patch('tripleo_heat_merge.merge.parse_scaling')
+    def test_generate_scaling_params(self, mock_parse_scaling):
+        # Setup
+        overcloud_roles = {'controller': 1, 'compute': 12}
+
+        # Test
+        template_tools.generate_scaling_params(overcloud_roles)
+
+        # Verify
+        mock_parse_scaling.assert_called_once_with(['NovaCompute=12'])
+
+    @mock.patch('tripleo_heat_merge.merge.merge')
+    def test_merge_templates(self, mock_merge):
+        # Setup
+        overcloud_roles = {'controller': 1, 'compute': 12}
+
+        # Test
+        template_tools.merge_templates(overcloud_roles)
+
+        # Verify
+        mock_merge.assert_called_once_with([
+            '/etc/tuskar/tripleo-heat-templates/overcloud-source.yaml',
+            '/etc/tuskar/tripleo-heat-templates/ssl-source.yaml',
+            '/etc/tuskar/tripleo-heat-templates/swift-source.yaml'],
+            None,
+            None,
+            scaling={
+                'NovaCompute0': 12
+            },
+            included_template_dir='/etc/tuskar/tripleo-heat-templates/'
+        )