From cff7420cfffce50cf70389d1808cbeea983973c9 Mon Sep 17 00:00:00 2001
From: Scott Hussey <sh8121@att.com>
Date: Mon, 2 Jul 2018 16:20:07 -0500
Subject: [PATCH] Support links for task status
- Some status changes in a task may have additional information that
is referenced by a URI link. Support describing these links and
returning them via API.
- Refactor alembic stuff to better handle table schema updates
- Add unit tests
Change-Id: Iae63a9716f2522578be0244925fc274a4338eac4
---
.../4713e7ebca9_add_task_status_links.py | 29 +++++
.../4a5bef3702b_create_build_data_table.py | 2 +-
...593a123e7c5_create_base_database_tables.py | 10 +-
drydock_provisioner/control/util.py | 30 +++++
drydock_provisioner/drydock_client/session.py | 5 +-
drydock_provisioner/objects/task.py | 36 ++++++
drydock_provisioner/statemgmt/db/tables.py | 29 ++++-
etc/drydock/drydock.conf.sample | 3 +
tests/unit/test_task_link.py | 114 ++++++++++++++++++
9 files changed, 244 insertions(+), 14 deletions(-)
create mode 100644 alembic/versions/4713e7ebca9_add_task_status_links.py
create mode 100644 drydock_provisioner/control/util.py
create mode 100644 tests/unit/test_task_link.py
diff --git a/alembic/versions/4713e7ebca9_add_task_status_links.py b/alembic/versions/4713e7ebca9_add_task_status_links.py
new file mode 100644
index 00000000..5318a55f
--- /dev/null
+++ b/alembic/versions/4713e7ebca9_add_task_status_links.py
@@ -0,0 +1,29 @@
+"""add task status links
+
+Revision ID: 4713e7ebca9
+Revises: 4a5bef3702b
+Create Date: 2018-07-05 14:54:18.381988
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4713e7ebca9'
+down_revision = '4a5bef3702b'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+from drydock_provisioner.statemgmt.db import tables
+
+
+def upgrade():
+ for c in tables.Tasks.__add_result_links__:
+ op.add_column(tables.Tasks.__tablename__, c)
+
+
+def downgrade():
+ for c in tables.Tasks.__add_result_links__:
+ op.drop_column(tables.Tasks.__tablename__, c.name)
+
diff --git a/alembic/versions/4a5bef3702b_create_build_data_table.py b/alembic/versions/4a5bef3702b_create_build_data_table.py
index 79e0ec46..8408bc25 100644
--- a/alembic/versions/4a5bef3702b_create_build_data_table.py
+++ b/alembic/versions/4a5bef3702b_create_build_data_table.py
@@ -19,7 +19,7 @@ from drydock_provisioner.statemgmt.db import tables
def upgrade():
op.create_table(tables.BuildData.__tablename__,
- *tables.BuildData.__schema__)
+ *tables.BuildData.__baseschema__)
def downgrade():
diff --git a/alembic/versions/9593a123e7c5_create_base_database_tables.py b/alembic/versions/9593a123e7c5_create_base_database_tables.py
index ed2c357a..16b1a013 100644
--- a/alembic/versions/9593a123e7c5_create_base_database_tables.py
+++ b/alembic/versions/9593a123e7c5_create_base_database_tables.py
@@ -18,15 +18,15 @@ from drydock_provisioner.statemgmt.db import tables
def upgrade():
- op.create_table(tables.Tasks.__tablename__, *tables.Tasks.__schema__)
+ op.create_table(tables.Tasks.__tablename__, *tables.Tasks.__baseschema__)
op.create_table(tables.ResultMessage.__tablename__,
- *tables.ResultMessage.__schema__)
+ *tables.ResultMessage.__baseschema__)
op.create_table(tables.ActiveInstance.__tablename__,
- *tables.ActiveInstance.__schema__)
+ *tables.ActiveInstance.__baseschema__)
op.create_table(tables.BootAction.__tablename__,
- *tables.BootAction.__schema__)
+ *tables.BootAction.__baseschema__)
op.create_table(tables.BootActionStatus.__tablename__,
- *tables.BootActionStatus.__schema__)
+ *tables.BootActionStatus.__baseschema__)
def downgrade():
diff --git a/drydock_provisioner/control/util.py b/drydock_provisioner/control/util.py
new file mode 100644
index 00000000..d5b9d840
--- /dev/null
+++ b/drydock_provisioner/control/util.py
@@ -0,0 +1,30 @@
+# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Reusable utility functions for API access."""
+from drydock_provisioner.error import ApiError
+from drydock_provisioner.drydock_client.session import KeystoneClient
+from drydock_provisioner.util import KeystoneUtils
+
+def get_internal_api_href(ver):
+ """Get the internal API href for Drydock API version ``ver``."""
+
+ # TODO(sh8121att) Support versioned service registration
+ supported_versions = ['v1.0']
+ if ver in supported_versions:
+ ks_sess = KeystoneUtils.get_session()
+ url = KeystoneClient.get_endpoint(
+ "physicalprovisioner", ks_sess=ks_sess, interface='internal')
+ return url
+ else:
+ raise ApiError("API version %s unknown." % ver)
diff --git a/drydock_provisioner/drydock_client/session.py b/drydock_provisioner/drydock_client/session.py
index cbb11950..55bf5006 100644
--- a/drydock_provisioner/drydock_client/session.py
+++ b/drydock_provisioner/drydock_client/session.py
@@ -180,12 +180,13 @@ class DrydockSession(object):
class KeystoneClient(object):
@staticmethod
- def get_endpoint(endpoint, ks_sess=None, auth_info=None):
+ def get_endpoint(endpoint, ks_sess=None, auth_info=None, interface='internal'):
"""
Wraps calls to keystone for lookup of an endpoint by service type
:param endpoint: The endpoint to look up
:param ks_sess: A keystone session to use for accessing endpoint catalogue
:param auth_info: Authentication info to use for building a token if a ``ks_sess`` is not specified
+ :param str interface: Which registered endpoint to return
:returns: The url string of the endpoint
:rtype: str
"""
@@ -193,7 +194,7 @@ class KeystoneClient(object):
ks_sess = KeystoneClient.get_ks_session(**auth_info)
return ks_sess.get_endpoint(
- interface='internal', service_type=endpoint)
+ interface=interface, service_type=endpoint)
@staticmethod
def get_token(ks_sess=None, auth_info=None):
diff --git a/drydock_provisioner/objects/task.py b/drydock_provisioner/objects/task.py
index 72253bf4..6332c663 100644
--- a/drydock_provisioner/objects/task.py
+++ b/drydock_provisioner/objects/task.py
@@ -371,6 +371,8 @@ class Task(object):
self.result.successes,
'result_failures':
self.result.failures,
+ 'result_links':
+ self.result.links,
'status':
self.status,
'created':
@@ -486,6 +488,7 @@ class Task(object):
i.result.status = d.get('result_status')
i.result.successes = d.get('result_successes', [])
i.result.failures = d.get('result_failures', [])
+ i.result.links = d.get('result_links', [])
# Deserialize the request context for this task
if i.request_context is not None:
@@ -506,6 +509,8 @@ class TaskStatus(object):
self.reason = None
self.status = hd_fields.ActionResult.Incomplete
+ self.links = dict()
+
# For tasks operating on multiple contexts (nodes, networks, etc...)
# track which contexts ended successfully and which failed
self.successes = []
@@ -515,6 +520,31 @@ class TaskStatus(object):
def obj_name(cls):
return cls.__name__
+ def add_link(self, relation, uri):
+ """Add a external reference link to this status.
+
+ :param str relation: The relation of the link
+ :param str uri: A valid URI that references the external content
+ """
+ self.links.setdefault(relation, [])
+ self.links[relation].append(uri)
+
+ def get_links(self, relation=None):
+ """Get one or more links of this status.
+
+ If ``relation`` is None, then return all links.
+
+ :param str relation: Return only links that exhibit this relation
+ :returns: a list of str URIs or empty list
+ """
+ if relation:
+ return self.links.get(relation, [])
+ else:
+ all_links = list()
+ for v in self.links.values():
+ all_links.extend(v)
+ return all_links
+
def set_message(self, msg):
self.message = msg
@@ -560,6 +590,11 @@ class TaskStatus(object):
return new_msg
def to_dict(self):
+ links = list()
+ if self.links:
+ for k, v in self.links.items():
+ for r in v:
+ links.append(dict(rel=k, href=r))
return {
'kind': 'Status',
'apiVersion': 'v1.0',
@@ -569,6 +604,7 @@ class TaskStatus(object):
'status': self.status,
'successes': self.successes,
'failures': self.failures,
+ 'links': links,
'details': {
'errorCount': self.error_count,
'messageList': [x.to_dict() for x in self.message_list],
diff --git a/drydock_provisioner/statemgmt/db/tables.py b/drydock_provisioner/statemgmt/db/tables.py
index 5568fbf2..39ce7815 100644
--- a/drydock_provisioner/statemgmt/db/tables.py
+++ b/drydock_provisioner/statemgmt/db/tables.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Definitions for Drydock database tables."""
+import copy
from sqlalchemy.schema import Table, Column
from sqlalchemy.types import Boolean, DateTime, String, Integer, Text
@@ -30,7 +31,7 @@ class Tasks(ExtendTable):
__tablename__ = 'tasks'
- __schema__ = [
+ __baseschema__ = [
Column('task_id', pg.BYTEA(16), primary_key=True),
Column('parent_task_id', pg.BYTEA(16)),
Column('subtask_id_list', pg.ARRAY(pg.BYTEA(16))),
@@ -54,13 +55,19 @@ class Tasks(ExtendTable):
Column('terminate', Boolean, default=False)
]
+ __add_result_links__ = [
+ Column('result_links', pg.JSON),
+ ]
+
+ __schema__ = copy.copy(__baseschema__)
+ __schema__.extend(__add_result_links__)
class ResultMessage(ExtendTable):
"""Table for tracking result/status messages."""
__tablename__ = 'result_message'
- __schema__ = [
+ __baseschema__ = [
Column('sequence', Integer, primary_key=True),
Column('task_id', pg.BYTEA(16)),
Column('message', String(1024)),
@@ -71,37 +78,43 @@ class ResultMessage(ExtendTable):
Column('extra', pg.JSON)
]
+ __schema__ = copy.copy(__baseschema__)
+
class ActiveInstance(ExtendTable):
"""Table to organize multiple orchestrator instances."""
__tablename__ = 'active_instance'
- __schema__ = [
+ __baseschema__ = [
Column('dummy_key', Integer, primary_key=True),
Column('identity', pg.BYTEA(16)),
Column('last_ping', DateTime),
]
+ __schema__ = copy.copy(__baseschema__)
+
class BootAction(ExtendTable):
"""Table persisting node build data."""
__tablename__ = 'boot_action'
- __schema__ = [
+ __baseschema__ = [
Column('node_name', String(280), primary_key=True),
Column('task_id', pg.BYTEA(16)),
Column('identity_key', pg.BYTEA(32)),
]
+ __schema__ = copy.copy(__baseschema__)
+
class BootActionStatus(ExtendTable):
"""Table tracking status of node boot actions."""
__tablename__ = 'boot_action_status'
- __schema__ = [
+ __baseschema__ = [
Column('node_name', String(280), index=True),
Column('action_id', pg.BYTEA(16), primary_key=True),
Column('action_name', String(64)),
@@ -110,13 +123,15 @@ class BootActionStatus(ExtendTable):
Column('action_status', String(32)),
]
+ __schema__ = copy.copy(__baseschema__)
+
class BuildData(ExtendTable):
"""Table for persisting node build data."""
__tablename__ = 'build_data'
- __schema__ = [
+ __baseschema__ = [
Column('node_name', String(32), index=True),
Column('task_id', pg.BYTEA(16), index=True),
Column('collected_date', DateTime),
@@ -124,3 +139,5 @@ class BuildData(ExtendTable):
Column('data_format', String(32)),
Column('data_element', Text),
]
+
+ __schema__ = copy.copy(__baseschema__)
diff --git a/etc/drydock/drydock.conf.sample b/etc/drydock/drydock.conf.sample
index a8fbb556..09eb4b9f 100644
--- a/etc/drydock/drydock.conf.sample
+++ b/etc/drydock/drydock.conf.sample
@@ -26,6 +26,9 @@
# The URI database connect string. (string value)
#database_connect_string = <None>
+# The SQLalchemy database connection pool size. (integer value)
+#pool_size = 15
+
[keystone_authtoken]
diff --git a/tests/unit/test_task_link.py b/tests/unit/test_task_link.py
new file mode 100644
index 00000000..2263e858
--- /dev/null
+++ b/tests/unit/test_task_link.py
@@ -0,0 +1,114 @@
+# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''Tests the functions for adding and retrieving task status links.'''
+from drydock_provisioner.objects import TaskStatus
+
+
+class TestTaskStatusLinks():
+ def test_links_add(self):
+ '''Add a link to a task status.'''
+ ts = TaskStatus()
+
+ relation = 'test'
+ uri = 'http://foo.com/test'
+
+ ts.add_link(relation, uri)
+
+ assert relation in ts.links
+ assert uri in ts.links.get(relation, [])
+
+ def test_links_get_empty(self):
+ '''Get links with an empty list.'''
+ ts = TaskStatus()
+
+ links = ts.get_links()
+
+ assert len(links) == 0
+
+ relation = 'test'
+ uri = 'http://foo.com/test'
+
+ ts.add_link(relation, uri)
+ links = ts.get_links(relation='none')
+
+ assert len(links) == 0
+
+ def test_links_get_all(self):
+ '''Get all links in a task status.'''
+ ts = TaskStatus()
+
+ relation = 'test'
+ uri = 'http://foo.com/test'
+
+ ts.add_link(relation, uri)
+ links = ts.get_links()
+
+ assert len(links) == 1
+ assert uri in links
+
+ def test_links_get_all_duplicate_relation(self):
+ '''Get all links where a relation has multiple uris.'''
+ ts = TaskStatus()
+
+ relation = 'test'
+ uri = 'http://foo.com/test'
+ uri2 = 'http://baz.com/test'
+
+ ts.add_link(relation, uri)
+ ts.add_link(relation, uri2)
+
+ links = ts.get_links()
+
+ assert len(links) == 2
+ assert uri in links
+ assert uri2 in links
+
+ def test_links_get_filter(self):
+ '''Get links with a filter.'''
+ ts = TaskStatus()
+
+ relation = 'test'
+ uri = 'http://foo.com/test'
+
+ relation2 = 'test2'
+ uri2 = 'http://baz.com/test'
+
+ ts.add_link(relation, uri)
+ ts.add_link(relation2, uri2)
+
+ links = ts.get_links(relation=relation)
+
+ assert len(links) == 1
+ assert uri in links
+
+ links = ts.get_links(relation=relation2)
+
+ assert len(links) == 1
+ assert uri2 in links
+
+ def test_links_serialization(self):
+ '''Check that task status serilization contains links correctly.'''
+ ts = TaskStatus()
+
+ relation = 'test'
+ uri = 'http://bar.com'
+
+ ts.set_message('foo')
+ ts.set_reason('bar')
+ ts.add_link(relation, uri)
+
+ ts_dict = ts.to_dict()
+
+ assert isinstance(ts_dict.get('links'), list)
+ assert {'rel': relation, 'href': uri} in ts_dict.get('links', [])