airship/drydock
Code Issues Proposed changes

Add build data for MAAS logs

Browse Source 
- Collect logs from MAAS when failures happen
  during deployment
- Save the logs to build data so it is available via
  API
- Add postgres integration test

Change-Id: Ied2d8539fe02a75f1f175a421b897b4f8ce07c8d
This commit is contained in:
Scott Hussey 2018-07-09 14:54:52 -05:00
parent eb996c27f4
commit f57301fae9
8 changed files with 178 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Makefile
View File

@ -51,7 +51,12 @@ coverage_test: build_drydock external_dep
# Run just unit tests
.PHONY: unit_tests
unit_tests: external_dep
tox -re py35
tox -re py35 $(TESTS)
# Run just DB integration tests
.PHONY: db_integration_tests
db_integration_tests: external_dep
tox -re integration $(TESTS)
# Freeze full set of Python requirements
.PHONY: req_freeze

3
drydock_provisioner/control/api.py
View File

@ -20,6 +20,7 @@ from .designs import DesignsPartsKindsResource
from .designs import DesignsPartResource
from .tasks import TasksResource
from .tasks import TaskResource
from .tasks import TaskBuilddataResource
from .nodes import NodesResource
from .nodes import NodeBuildDataResource
from .nodes import NodeFilterResource
@ -67,6 +68,8 @@ def start_api(state_manager=None, ingester=None, orchestrator=None):
TasksResource(state_manager=state_manager,
orchestrator=orchestrator)),
('/tasks/{task_id}', TaskResource(state_manager=state_manager)),
('/tasks/{task_id}/builddata',
TaskBuilddataResource(state_manager=state_manager)),
# API for managing site design data
('/designs', DesignsResource(state_manager=state_manager)),

17
drydock_provisioner/control/tasks.py
View File

@ -390,3 +390,20 @@ class TaskResource(StatefulResource):
# Finished this layer, incrementing for the next while loop.
current_layer = current_layer + 1
return resp_data, errors
class TaskBuilddataResource(StatefulResource):
"""Handler resource for /tasks/<id>/builddata singleton endpoint."""
@policy.ApiEnforcer('physical_provisioner:read_build_data')
def on_get(self, req, resp, task_id):
try:
bd_list = self.state_manager.get_build_data(task_id=task_id)
if not bd_list:
resp.status = falcon.HTTP_404
return
resp.body = json.dumps(bd_list)
except Exception as e:
resp.body = "Unexpected error."
resp.status = falcon.HTTP_500
resp.status = falcon.HTTP_200

2
drydock_provisioner/control/util.py
View File

@ -16,11 +16,13 @@ from drydock_provisioner.error import ApiError
from drydock_provisioner.drydock_client.session import KeystoneClient
from drydock_provisioner.util import KeystoneUtils
def get_internal_api_href(ver):
"""Get the internal API href for Drydock API version ``ver``."""
# TODO(sh8121att) Support versioned service registration
supported_versions = ['v1.0']
if ver in supported_versions:
ks_sess = KeystoneUtils.get_session()
url = KeystoneClient.get_endpoint(

45
drydock_provisioner/drivers/node/maasdriver/actions/node.py
View File

@ -27,6 +27,7 @@ import drydock_provisioner.objects.fields as hd_fields
import drydock_provisioner.objects.hostprofile as hostprofile
import drydock_provisioner.objects as objects
from drydock_provisioner.control.util import get_internal_api_href
from drydock_provisioner.orchestrator.actions.orchestrator import BaseAction
import drydock_provisioner.drivers.node.maasdriver.models.fabric as maas_fabric
@ -51,6 +52,22 @@ class BaseMaasAction(BaseAction):
self.logger = logging.getLogger(
config.config_mgr.conf.logging.nodedriver_logger_name)
def _add_detail_logs(self, node, machine, data_gen, result_type='all'):
result_details = machine.get_task_results(result_type=result_type)
for r in result_details:
bd = objects.BuildData(
node_name=node.name,
task_id=self.task.task_id,
collected_date=r.updated,
generator=data_gen,
data_format='text/plain',
data_element=r.get_decoded_data())
self.state_manager.post_build_data(bd)
log_href = "%s/tasks/%s/builddata" % (
get_internal_api_href("v1.0"), str(self.task.task_id))
self.task.result.add_link('detail_logs', log_href)
self.task.save()
class ValidateNodeServices(BaseMaasAction):
"""Action to validate MaaS is available and ready for use."""
@ -983,6 +1000,25 @@ class ConfigureHardware(BaseMaasAction):
ctx_type='node')
self.task.success(focus=n.get_id())
self.collect_build_data(machine)
else:
msg = "Node %s failed commissioning." % (n.name)
self.logger.info(msg)
self.task.add_status_msg(
msg=msg,
error=True,
ctx=n.name,
ctx_type='node')
self.task.failure(focus=n.get_id())
self._add_detail_logs(
n,
machine,
'maas_commission_log',
result_type='commissioning')
self._add_detail_logs(
n,
machine,
'maas_testing_log',
result_type='testing')
elif machine.status_name in ['Commissioning', 'Testing']:
msg = "Located node %s in MaaS, node already being commissioned. Skipping..." % (
n.name)
@ -2123,6 +2159,12 @@ class DeployNode(BaseMaasAction):
self.task.add_status_msg(
msg=msg, error=False, ctx=n.name, ctx_type='node')
self.task.success(focus=n.get_id())
elif machine.status_name.startswith('Failed'):
msg = "Node %s deployment failed" % (n.name)
self.logger.info(msg)
self.task.add_status_msg(
msg=msg, error=True, ctx=n.name, ctx_type='node')
self.task.failure(focus=n.get_id())
else:
msg = "Node %s deployment timed out" % (n.name)
self.logger.warning(msg)
@ -2130,6 +2172,9 @@ class DeployNode(BaseMaasAction):
msg=msg, error=True, ctx=n.name, ctx_type='node')
self.task.failure(focus=n.get_id())
self._add_detail_logs(
n, machine, 'maas_deploy_log', result_type='deploy')
self.task.set_status(hd_fields.TaskStatus.Complete)
self.task.save()

4
drydock_provisioner/drivers/node/maasdriver/models/machine.py
View File

@ -304,7 +304,9 @@ class Machine(model_base.ResourceBase):
``all``, ``commissioning``, ``testing``, ``deploy``
"""
node_results = maas_nr.NodeResults(
system_id_list=[self.resource_id], result_type=result_type)
self.api_client,
system_id_list=[self.resource_id],
result_type=result_type)
node_results.refresh()
return node_results

85
tests/integration/postgres/test_noderesult_links.py Normal file
View File

@ -0,0 +1,85 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test build data collection and persistence."""
from drydock_provisioner.objects import fields as hd_fields
from drydock_provisioner.drivers.node.maasdriver.actions.node import BaseMaasAction
from drydock_provisioner.drivers.node.maasdriver.models.machine import Machine
class TestNodeResultLinks(object):
def test_create_detail_log_links(self, setup, blank_state, mocker,
input_files, deckhand_orchestrator):
"""Test that the detail log collection from MaaS works."""
class MockedResponse():
status_code = 200
def json(self):
resp_content = [{
"id":
3,
"data":
"SGVsbG8gV29ybGQh",
"result_type":
0,
"script_result":
0,
"resource_uri":
"/MAAS/api/2.0/commissioning-scripts/",
"updated":
"2018-07-06T14:32:20.129",
"node": {
"system_id": "r7mqnw"
},
"created":
"2018-07-06T14:37:12.632",
"name":
"hello_world"
}]
return resp_content
input_file = input_files.join("deckhand_fullsite.yaml")
design_ref = "file://%s" % str(input_file)
task = deckhand_orchestrator.create_task(
action=hd_fields.OrchestratorAction.Noop, design_ref=design_ref)
task.set_status(hd_fields.TaskStatus.Running)
task.save()
api_client = mocker.MagicMock()
api_client.get.return_value = MockedResponse()
machine = Machine(api_client)
machine.resource_id = 'r7mqnw'
node = mocker.MagicMock()
node.configure_mock(name='n1')
action = BaseMaasAction(task, deckhand_orchestrator, blank_state)
with mocker.patch(
'drydock_provisioner.drivers.node.maasdriver.actions.node.get_internal_api_href',
mocker.MagicMock(return_value='http://drydock/api/v1.0')):
action._add_detail_logs(node, machine, 'hello_world')
bd = blank_state.get_build_data(task_id=task.task_id)
assert len(bd) == 1
links_list = task.result.get_links()
assert len(links_list) > 0
for l in links_list:
assert str(task.task_id) in l

25
tests/unit/test_maasdriver_noderesults.py
View File

@ -18,6 +18,7 @@ from drydock_provisioner.drivers.node.maasdriver.models.node_results import Node
class TestMaasNodeResults():
def test_get_noderesults(self, mocker):
'''Test noderesults refresh call to load a list of NodeResults.'''
# A object to return that looks like a requests response
# object wrapping a MAAS API response
class MockedResponse():
@ -26,17 +27,25 @@ class TestMaasNodeResults():
def json(self):
resp_content = [{
"id": 3,
"data": "SGVsbG8gV29ybGQh",
"result_type": 0,
"script_result": 0,
"resource_uri": "/MAAS/api/2.0/commissioning-scripts/",
"updated": "2018-07-06T14:32:20.129",
"id":
3,
"data":
"SGVsbG8gV29ybGQh",
"result_type":
0,
"script_result":
0,
"resource_uri":
"/MAAS/api/2.0/commissioning-scripts/",
"updated":
"2018-07-06T14:32:20.129",
"node": {
"system_id": "r7mqnw"
},
"created": "2018-07-06T14:37:12.632",
"name": "hello_world"
"created":
"2018-07-06T14:37:12.632",
"name":
"hello_world"
}]
return resp_content