x/python-hnvclient
Code Issues Proposed changes

Add base data model in order to improve data representation

Browse Source 
The proposed data model improves the way how the information
is structured and allows related entities to to inherit data.
This commit is contained in:
Alexandru Coman 2017-01-09 16:02:03 +02:00
parent ca738d46ee
commit 6251ca5344
No known key found for this signature in database
GPG Key ID: A7B6A9021F704507
8 changed files with 631 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.gitignore vendored Normal file
View File

@ -0,0 +1,55 @@
*.py[cod]
# C extensions
*.so
# Packages
*.egg
*.eggs
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
# Installer logs
pip-log.txt
# Unit test / coverage reports
.coverage
.tox
nosetests.xml
.testrepository
.venv
cover
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Complexity
output/*.html
output/*/index.html
# Sphinx
doc/build
# pbr generates these
AUTHORS
ChangeLog
# Editors
*~
.*.swp
.*sw?

10
.pylintrc
View File

@ -13,7 +13,7 @@ ignore=CVS
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.
ignore-patterns=
ignore-patterns=test_*
# Pickle collected data for later comparisons.
persistent=yes
@ -166,12 +166,12 @@ ignore-mixin-members=yes
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=hnv_client.tests
ignored-modules=
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local,_thread._local
ignored-classes=optparse.Values,thread._local,_thread._local, hnv_client.common.model.Model
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
@ -366,7 +366,7 @@ analyse-fallback-blocks=no
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
max-args=15
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
@ -391,7 +391,7 @@ max-parents=7
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
min-public-methods=1
# Maximum number of public methods for a class (see R0904).
max-public-methods=20

15
hnv_client/common/constant.py
View File

@ -0,0 +1,15 @@
# Copyright 2017 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Shared constants across the bcbio-nextgen-vm project."""

7
hnv_client/common/exception.py
View File

@ -51,6 +51,13 @@ class HNVException(Exception):
super(HNVException, self).__init__(message)
class DataProcessingError(HNVException):
"""Base exception class for data processing related errors."""
template = "The provided information is incomplete or invalid."
class NotFound(HNVException):
"""The required object is not available in container."""

308
hnv_client/common/model.py Normal file
View File

@ -0,0 +1,308 @@
# Copyright 2017 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""This module contains data models used across the project."""
# pylint: disable=protected-access
import copy
from oslo_log import log as logging
import six
from hnv_client.common import exception
LOG = logging.getLogger(__name__)
class _FieldDescriptor(object):
"""Descriptor for all the available fields for a model.
Fields are exposed as descriptors in order to control access to the
underlying raw data.
"""
def __init__(self, field):
self._field = field
self._attribute = field.key
@property
def field(self):
"""Expose the received field object."""
return self._field
def __get__(self, instance, instance_type=None):
if instance is not None:
return instance._data.get(self._attribute)
return self._field
def __set__(self, instance, value):
if instance.provision_done:
if self._field.is_read_only:
raise TypeError("%r does not support item assignment" %
self._field.name)
# Update the value of the field
instance._data[self._attribute] = value
# Keep track of the changes
instance._changes[self._attribute] = value
class Field(object):
"""Meta information regarding the data components.
:param name: The name of the current piece of information.
:param key: The internal name for the current piece of
information.
:param default: Default value for the current field.
(default: `None`)
:param is_required: Whether the current piece of information is required
for the container object or can be missing.
(default: `False`)
:param is_property: Whether the current piece of information is a
propriety of the model. (default: `True`)
:param is_read_only: Whether the current piece of information can
be updated. (Default: `False`)
"""
def __init__(self, name, key, default=None, is_required=False,
is_property=True, is_read_only=False):
self._name = name
self._key = key
self._default = default
self._is_required = is_required
self._is_property = is_property
self._is_read_only = is_read_only
@property
def name(self):
"""The name of the current field."""
return self._name
@property
def key(self):
"""The internal name of the current field."""
return self._key
@property
def default(self):
"""Default value for the current field."""
return self._default
@property
def is_required(self):
"""Whether the current field is required or can be missing."""
return self._is_required
@property
def is_property(self):
"""Whether the current field is a model property."""
return self._is_property
@property
def is_read_only(self):
"""Whether the current field can be updated."""
return self._is_read_only
def add_to_class(self, model_class):
"""Replace the `Field` attribute with a named `_FieldDescriptor`.
.. note::
This method is called during construction of the `Model`.
"""
model_class._meta.add_field(self)
setattr(model_class, self.name, _FieldDescriptor(self))
class _ModelOptions(object):
"""Container for all the model options.
.. note::
The current object will be created by the model metaclass.
"""
def __init__(self, cls):
self._model_class = cls
self._name = cls.__name__
self._fields = {}
self._defaults = {}
self._default_callables = {}
@property
def fields(self):
"""All the available fields for the current model."""
return self._fields
def add_field(self, field):
"""Add the received field to the model."""
self.remove_field(field.name)
self._fields[field.name] = field
if field.default is not None:
if six.callable(field.default):
self._default_callables[field.key] = field.default
else:
self._defaults[field.key] = field.default
def remove_field(self, field_name):
"""Remove the field with the received field name from model."""
field = self._fields.pop(field_name, None)
if field is not None and field.default is not None:
if six.callable(field.default):
self._default_callables.pop(field.name, None)
else:
self._defaults.pop(field.name, None)
def get_defaults(self):
"""Get a dictionary that contains all the available defaults."""
defaults = self._defaults.copy()
for field_name, default in self._default_callables.items():
defaults[field_name] = default()
return defaults
class _BaseModel(type):
"""Metaclass used for properly setting up a new model."""
def __new__(mcs, name, bases, attrs):
# The inherit is made by deep copying the underlying field into
# the attributes of the new model.
for base in bases:
for key, attribute in base.__dict__.items():
if key not in attrs and isinstance(attribute,
_FieldDescriptor):
attrs[key] = copy.deepcopy(attribute.field)
# Initialize the new class and set the magic attributes
cls = super(_BaseModel, mcs).__new__(mcs, name, bases, attrs)
# Create the _ModelOptions object and inject it in the new class
setattr(cls, "_meta", _ModelOptions(cls))
# Get all the available fields for the current model.
for name, field in list(cls.__dict__.items()):
if isinstance(field, Field) and not name.startswith("_"):
field.add_to_class(cls)
# Create string representation for the current model before finalizing
setattr(cls, '__str__', lambda self: '%s' % cls.__name__)
return cls
@six.add_metaclass(_BaseModel)
class Model(object):
"""Container for meta information regarding the data structure."""
def __init__(self, **fields):
self._data = self._meta.get_defaults()
self._changes = {}
self._provision_done = False
for field in self._meta.fields.values():
value = fields.pop(field.name, None)
if field.key not in self._data or value:
setattr(self, field.name, value)
if fields:
LOG.debug("Unrecognized fields: %r", fields)
self._provision_done = True
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
my_properties = self.dump().get("properties", {})
other_properties = other.dump().get("properties", {})
return my_properties == other_properties
def __ne__(self, other):
return not self.__eq__(other)
@classmethod
def from_raw_data(cls, raw_data):
"""Create a new model using raw API response."""
content = {}
properties = raw_data.pop("properties", {})
for field_name, field in cls._meta.fields.items():
if field.is_property:
value = properties.pop(field.key, None)
else:
value = raw_data.pop(field.key, None)
content[field_name] = value
if raw_data:
LOG.debug("Unrecognized fields: %r", raw_data)
if properties:
LOG.debug("Unrecognized properties: %r", properties)
return cls(**content)
@property
def provision_done(self):
"""Whether the creation of the model is complete."""
return self._provision_done
def validate(self):
"""Check if the current model was properly created."""
for field_name, field in self._meta.fields.items():
if field.is_required and self._data.get(field.name) is None:
raise exception.DataProcessingError(
"The required field %(field)r is missing.",
field=field_name)
def update(self, fields=None):
"""Update the value of one or more fields."""
for field_name, field in self._meta.fields.items():
if field_name in fields:
self._changes[field.key] = fields[field_name]
self._data.update(self._changes)
def commit(self, wait=False, timeout=None):
"""Apply all the changes on the current model."""
# pylint: disable=unused-argument
self._data.update(self._changes)
self._changes.clear()
def dump(self, include_read_only=True):
"""Create a dictionary with the content of the current model."""
content = {}
for field in self._meta.fields.values():
if field.is_read_only and not include_read_only:
continue
value = self._data.get(field.key)
if isinstance(value, Model):
# The raw content of the model is required
value = value.dump()
if not field.is_required and value is None:
# The value of this field is not relevant
continue
if field.is_property:
# The current field is a property and its value should
# be stored into the `properties` key.
properties = content.setdefault("properties", {})
properties[field.key] = value
else:
content[field.key] = value
return content

0
hnv_client/tests/common/__init__.py Normal file
View File

238
hnv_client/tests/common/test_model.py Normal file
View File

@ -0,0 +1,238 @@
# Copyright 2017 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# pylint: disable=protected-access, missing-docstring
import unittest
try:
import unittest.mock as mock
except ImportError:
import mock
from hnv_client.common import exception
from hnv_client.common import model
class TestFieldDescriptor(unittest.TestCase):
def test_field_access(self):
instance = mock.Mock()
field = mock.Mock()
field_descriptor = model._FieldDescriptor(field)
self.assertIs(field, field_descriptor.__get__(None))
field_descriptor.__get__(instance)
instance._data.get.assert_called_once_with(field.key)
def test_set_field(self):
instance = mock.MagicMock()
instance._changes = {}
field = mock.Mock()
field.is_read_only = False
field_descriptor = model._FieldDescriptor(field)
field_descriptor.__set__(instance, mock.sentinel.value)
self.assertIs(instance._changes[field.key], mock.sentinel.value)
def test_set_read_only_field(self):
instance = mock.MagicMock()
field = mock.Mock()
field.is_read_only = True
field_descriptor = model._FieldDescriptor(field)
with self.assertRaises(TypeError):
field_descriptor.__set__(instance, mock.sentinel.value)
def test_field_property(self):
field = mock.Mock()
field_descriptor = model._FieldDescriptor(field)
self.assertIs(field_descriptor.field, field)
class TestField(unittest.TestCase):
def test_properties(self):
field = model.Field(name=mock.sentinel.name,
key=mock.sentinel.name,
default=mock.sentinel.default,
is_required=mock.sentinel.is_required,
is_property=mock.sentinel.is_property,
is_read_only=mock.sentinel.is_read_only)
self.assertIs(field.name, mock.sentinel.name)
self.assertIs(field.default, mock.sentinel.default)
self.assertIs(field.is_required, mock.sentinel.is_required)
self.assertIs(field.is_property, mock.sentinel.is_property)
self.assertIs(field.is_read_only, mock.sentinel.is_read_only)
@mock.patch("hnv_client.common.model._FieldDescriptor")
def test_add_to_class(self, mock_field_descriptor):
field = model.Field(name="test_add_to_class", key="test")
model_class = mock.Mock()
field.add_to_class(model_class)
mock_field_descriptor.assert_called_once_with(field)
self.assertIsNotNone(getattr(model_class, "test_add_to_class"))
class TestModelOptions(unittest.TestCase):
def test_initialization(self):
mock.sentinel.cls.__name__ = mock.sentinel.cls.name
model_options = model._ModelOptions(cls=mock.sentinel.cls)
self.assertIs(model_options._model_class, mock.sentinel.cls)
self.assertEqual(model_options._name, mock.sentinel.cls.name)
@mock.patch("six.callable")
@mock.patch("hnv_client.common.model._ModelOptions.remove_field")
def _test_add_field(self, mock_remove_field, mock_callable,
callable_default):
model_options = model._ModelOptions(self.__class__)
test_field = model.Field(name=mock.sentinel.name,
key=mock.sentinel.key,
is_required=False,
default=mock.sentinel.default)
mock_callable.return_value = callable_default
model_options.add_field(test_field)
mock_remove_field.assert_called_once_with(mock.sentinel.name)
self.assertIs(model_options._fields[test_field.name], test_field)
if callable_default:
self.assertIs(model_options._default_callables[test_field.key],
mock.sentinel.default)
else:
self.assertIs(model_options._defaults[test_field.key],
mock.sentinel.default)
def test_add_field(self):
self._test_add_field(callable_default=False)
self._test_add_field(callable_default=True)
@mock.patch("six.callable")
def _test_remove_field(self, mock_callable, callable_default):
mock_callable.return_value = callable_default
model_options = model._ModelOptions(self.__class__)
test_field = model.Field(name=mock.sentinel.name,
key="test_field",
is_required=False,
default=mock.sentinel.default)
model_options.add_field(test_field)
model_options.remove_field(test_field.name)
self.assertNotIn(test_field.name, model_options._fields)
if callable_default:
self.assertNotIn(test_field.name,
model_options._default_callables)
else:
self.assertNotIn(test_field.name, model_options._defaults)
def test_remove_field(self):
self._test_remove_field(callable_default=False)
self._test_remove_field(callable_default=True)
def test_get_defaults(self):
test_field = model.Field(
name=mock.sentinel.name, key=mock.sentinel.key,
is_required=False, default=lambda: mock.sentinel.default)
model_options = model._ModelOptions(self.__class__)
model_options.add_field(test_field)
defaults = model_options.get_defaults()
self.assertEqual(defaults, {mock.sentinel.key: mock.sentinel.default})
class TestBaseModel(unittest.TestCase):
def test_create_model(self):
class _Test(model.Model):
field1 = model.Field(name="field1", key="field1", default=1)
self.assertTrue(hasattr(_Test, "_meta"))
self.assertEqual(_Test().field1, 1)
def test_inherit_fields(self):
class _TestBase(model.Model):
field1 = model.Field(name="field1", key="key1",
default=1)
field2 = model.Field(name="field2", key="key2")
class _Test(_TestBase):
field2 = model.Field(name="field2", key="key2",
default=2)
class _FinalTest(_Test):
field3 = model.Field(name="field3", key="key3",
is_required=True)
final_test = _FinalTest(field3=3)
self.assertEqual(final_test.field1, 1)
self.assertEqual(final_test.field2, 2)
self.assertEqual(final_test.field3, 3)
class TestModel(unittest.TestCase):
class _Test(model.Model):
field1 = model.Field(name="field1", key="key1",
is_required=True,
is_property=False)
field2 = model.Field(name="field2", key="key2",
is_required=False,
is_property=False)
field3 = model.Field(name="field3", key="key3",
is_required=True,
is_property=True)
def setUp(self):
self._raw_data = {"key1": 1, "key2": 2, "properties": {"key3": 3}}
def test_model_eq(self):
test = self._Test(field1=1, field3=2)
test2 = self._Test(field1=1, field3=3)
self.assertFalse(test == self)
self.assertTrue(test == test)
self.assertFalse(test == test2)
def test_validate(self):
test = self._Test(field1=1, key="test_validate")
self.assertRaises(exception.DataProcessingError, test.validate)
def test_update(self):
test = self._Test(field1="change_me", field2="change_me")
test.update({"field1": mock.sentinel.field1,
"field2": mock.sentinel.field2})
self.assertIs(test.field1, mock.sentinel.field1)
self.assertIs(test.field2, mock.sentinel.field2)
def test_dump(self):
test = self._Test(field1=1, field2=2, field3=3)
self.assertEqual(test.dump(), self._raw_data)
def test_from_raw_data(self):
test = self._Test.from_raw_data(self._raw_data)
self.assertEqual(test.field1, 1)
self.assertEqual(test.field2, 2)
self.assertEqual(test.field3, 3)

3
requirements.txt
View File

@ -1,4 +1,7 @@
pbr>=1.8
six>=1.7.0
oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
oslo.log>=3.11.0 # Apache-2.0
requests
requests_ntlm