x/python-hnvclient
Code Issues Proposed changes

Merge pull request #1 from alexcoman/feature/data-model

Browse Source 
Add base data model in order to improve data representation
This commit is contained in:
Alexandru Coman 2017-01-30 14:06:51 +02:00 committed by GitHub
commit c20790a43e
8 changed files with 631 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.gitignore vendored Normal file
View File

@ -0,0 +1,55 @@
*.py[cod]
# C extensions
*.so
# Packages
*.egg
*.eggs
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
# Installer logs
pip-log.txt
# Unit test / coverage reports
.coverage
.tox
nosetests.xml
.testrepository
.venv
cover
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Complexity
output/*.html
output/*/index.html
# Sphinx
doc/build
# pbr generates these
AUTHORS
ChangeLog
# Editors
*~
.*.swp
.*sw?

10
.pylintrc
View File

@ -13,7 +13,7 @@ ignore=CVS
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.
ignore-patterns=
ignore-patterns=test_*
# Pickle collected data for later comparisons.
persistent=yes
@ -166,12 +166,12 @@ ignore-mixin-members=yes
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=hnv_client.tests
ignored-modules=
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local,_thread._local
ignored-classes=optparse.Values,thread._local,_thread._local, hnv_client.common.model.Model
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
@ -366,7 +366,7 @@ analyse-fallback-blocks=no
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
max-args=15
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
@ -391,7 +391,7 @@ max-parents=7
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
min-public-methods=1
# Maximum number of public methods for a class (see R0904).
max-public-methods=20

15
hnv_client/common/constant.py
View File

@ -0,0 +1,15 @@
# Copyright 2017 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Shared constants across the bcbio-nextgen-vm project."""

7
hnv_client/common/exception.py
View File

@ -51,6 +51,13 @@ class HNVException(Exception):
super(HNVException, self).__init__(message)
class DataProcessingError(HNVException):
"""Base exception class for data processing related errors."""
template = "The provided information is incomplete or invalid."
class NotFound(HNVException):
"""The required object is not available in container."""

308
hnv_client/common/model.py Normal file
View File

@ -0,0 +1,308 @@
# Copyright 2017 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""This module contains data models used across the project."""
# pylint: disable=protected-access
import copy
from oslo_log import log as logging
import six
from hnv_client.common import exception
LOG = logging.getLogger(__name__)
class _FieldDescriptor(object):
"""Descriptor for all the available fields for a model.
Fields are exposed as descriptors in order to control access to the
underlying raw data.
"""
def __init__(self, field):
self._field = field
self._attribute = field.key
@property
def field(self):
"""Expose the received field object."""
return self._field
def __get__(self, instance, instance_type=None):
if instance is not None:
return instance._data.get(self._attribute)
return self._field
def __set__(self, instance, value):
if instance.provision_done:
if self._field.is_read_only:
raise TypeError("%r does not support item assignment" %
self._field.name)
# Update the value of the field
instance._data[self._attribute] = value
# Keep track of the changes
instance._changes[self._attribute] = value
class Field(object):
"""Meta information regarding the data components.
:param name: The name of the current piece of information.
:param key: The internal name for the current piece of
information.
:param default: Default value for the current field.
(default: `None`)
:param is_required: Whether the current piece of information is required
for the container object or can be missing.
(default: `False`)
:param is_property: Whether the current piece of information is a
propriety of the model. (default: `True`)
:param is_read_only: Whether the current piece of information can
be updated. (Default: `False`)
"""
def __init__(self, name, key, default=None, is_required=False,
is_property=True, is_read_only=False):
self._name = name
self._key = key
self._default = default
self._is_required = is_required
self._is_property = is_property
self._is_read_only = is_read_only
@property
def name(self):
"""The name of the current field."""
return self._name
@property
def key(self):
"""The internal name of the current field."""
return self._key
@property
def default(self):
"""Default value for the current field."""
return self._default
@property
def is_required(self):
"""Whether the current field is required or can be missing."""
return self._is_required
@property
def is_property(self):
"""Whether the current field is a model property."""
return self._is_property
@property
def is_read_only(self):
"""Whether the current field can be updated."""
return self._is_read_only
def add_to_class(self, model_class):
"""Replace the `Field` attribute with a named `_FieldDescriptor`.
.. note::
This method is called during construction of the `Model`.
"""
model_class._meta.add_field(self)
setattr(model_class, self.name, _FieldDescriptor(self))
class _ModelOptions(object):
"""Container for all the model options.
.. note::
The current object will be created by the model metaclass.
"""
def __init__(self, cls):
self._model_class = cls
self._name = cls.__name__
self._fields = {}
self._defaults = {}
self._default_callables = {}
@property
def fields(self):
"""All the available fields for the current model."""
return self._fields
def add_field(self, field):
"""Add the received field to the model."""
self.remove_field(field.name)
self._fields[field.name] = field
if field.default is not None:
if six.callable(field.default):
self._default_callables[field.key] = field.default
else:
self._defaults[field.key] = field.default
def remove_field(self, field_name):
"""Remove the field with the received field name from model."""
field = self._fields.pop(field_name, None)
if field is not None and field.default is not None:
if six.callable(field.default):
self._default_callables.pop(field.name, None)
else:
self._defaults.pop(field.name, None)
def get_defaults(self):
"""Get a dictionary that contains all the available defaults."""
defaults = self._defaults.copy()
for field_name, default in self._default_callables.items():
defaults[field_name] = default()
return defaults
class _BaseModel(type):
"""Metaclass used for properly setting up a new model."""
def __new__(mcs, name, bases, attrs):
# The inherit is made by deep copying the underlying field into
# the attributes of the new model.
for base in bases:
for key, attribute in base.__dict__.items():
if key not in attrs and isinstance(attribute,
_FieldDescriptor):
attrs[key] = copy.deepcopy(attribute.field)
# Initialize the new class and set the magic attributes
cls = super(_BaseModel, mcs).__new__(mcs, name, bases, attrs)
# Create the _ModelOptions object and inject it in the new class
setattr(cls, "_meta", _ModelOptions(cls))
# Get all the available fields for the current model.
for name, field in list(cls.__dict__.items()):
if isinstance(field, Field) and not name.startswith("_"):
field.add_to_class(cls)
# Create string representation for the current model before finalizing
setattr(cls, '__str__', lambda self: '%s' % cls.__name__)
return cls
@six.add_metaclass(_BaseModel)
class Model(object):
"""Container for meta information regarding the data structure."""
def __init__(self, **fields):
self._data = self._meta.get_defaults()
self._changes = {}
self._provision_done = False
for field in self._meta.fields.values():
value = fields.pop(field.name, None)
if field.key not in self._data or value:
setattr(self, field.name, value)
if fields:
LOG.debug("Unrecognized fields: %r", fields)
self._provision_done = True
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
my_properties = self.dump().get("properties", {})
other_properties = other.dump().get("properties", {})
return my_properties == other_properties
def __ne__(self, other):
return not self.__eq__(other)
@classmethod
def from_raw_data(cls, raw_data):
"""Create a new model using raw API response."""
content = {}
properties = raw_data.pop("properties", {})
for field_name, field in cls._meta.fields.items():
if field.is_property:
value = properties.pop(field.key, None)
else:
value = raw_data.pop(field.key, None)
content[field_name] = value
if raw_data:
LOG.debug("Unrecognized fields: %r", raw_data)
if properties:
LOG.debug("Unrecognized properties: %r", properties)
return cls(**content)
@property
def provision_done(self):
"""Whether the creation of the model is complete."""
return self._provision_done
def validate(self):
"""Check if the current model was properly created."""
for field_name, field in self._meta.fields.items():
if field.is_required and self._data.get(field.name) is None:
raise exception.DataProcessingError(
"The required field %(field)r is missing.",
field=field_name)
def update(self, fields=None):
"""Update the value of one or more fields."""
for field_name, field in self._meta.fields.items():
if field_name in fields:
self._changes[field.key] = fields[field_name]
self._data.update(self._changes)
def commit(self, wait=False, timeout=None):
"""Apply all the changes on the current model."""
# pylint: disable=unused-argument
self._data.update(self._changes)
self._changes.clear()
def dump(self, include_read_only=True):
"""Create a dictionary with the content of the current model."""
content = {}
for field in self._meta.fields.values():
if field.is_read_only and not include_read_only:
continue
value = self._data.get(field.key)
if isinstance(value, Model):
# The raw content of the model is required
value = value.dump()
if not field.is_required and value is None:
# The value of this field is not relevant
continue
if field.is_property:
# The current field is a property and its value should
# be stored into the `properties` key.
properties = content.setdefault("properties", {})
properties[field.key] = value
else:
content[field.key] = value
return content

0
hnv_client/tests/common/__init__.py Normal file
View File

238
hnv_client/tests/common/test_model.py Normal file
View File

@ -0,0 +1,238 @@
# Copyright 2017 Cloudbase Solutions Srl
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# pylint: disable=protected-access, missing-docstring
import unittest
try:
import unittest.mock as mock
except ImportError:
import mock
from hnv_client.common import exception
from hnv_client.common import model
class TestFieldDescriptor(unittest.TestCase):
def test_field_access(self):
instance = mock.Mock()
field = mock.Mock()
field_descriptor = model._FieldDescriptor(field)
self.assertIs(field, field_descriptor.__get__(None))
field_descriptor.__get__(instance)
instance._data.get.assert_called_once_with(field.key)
def test_set_field(self):
instance = mock.MagicMock()
instance._changes = {}
field = mock.Mock()
field.is_read_only = False
field_descriptor = model._FieldDescriptor(field)
field_descriptor.__set__(instance, mock.sentinel.value)
self.assertIs(instance._changes[field.key], mock.sentinel.value)
def test_set_read_only_field(self):
instance = mock.MagicMock()
field = mock.Mock()
field.is_read_only = True
field_descriptor = model._FieldDescriptor(field)
with self.assertRaises(TypeError):
field_descriptor.__set__(instance, mock.sentinel.value)
def test_field_property(self):
field = mock.Mock()
field_descriptor = model._FieldDescriptor(field)
self.assertIs(field_descriptor.field, field)
class TestField(unittest.TestCase):
def test_properties(self):
field = model.Field(name=mock.sentinel.name,
key=mock.sentinel.name,
default=mock.sentinel.default,
is_required=mock.sentinel.is_required,
is_property=mock.sentinel.is_property,
is_read_only=mock.sentinel.is_read_only)
self.assertIs(field.name, mock.sentinel.name)
self.assertIs(field.default, mock.sentinel.default)
self.assertIs(field.is_required, mock.sentinel.is_required)
self.assertIs(field.is_property, mock.sentinel.is_property)
self.assertIs(field.is_read_only, mock.sentinel.is_read_only)
@mock.patch("hnv_client.common.model._FieldDescriptor")
def test_add_to_class(self, mock_field_descriptor):
field = model.Field(name="test_add_to_class", key="test")
model_class = mock.Mock()
field.add_to_class(model_class)
mock_field_descriptor.assert_called_once_with(field)
self.assertIsNotNone(getattr(model_class, "test_add_to_class"))
class TestModelOptions(unittest.TestCase):
def test_initialization(self):
mock.sentinel.cls.__name__ = mock.sentinel.cls.name
model_options = model._ModelOptions(cls=mock.sentinel.cls)
self.assertIs(model_options._model_class, mock.sentinel.cls)
self.assertEqual(model_options._name, mock.sentinel.cls.name)
@mock.patch("six.callable")
@mock.patch("hnv_client.common.model._ModelOptions.remove_field")
def _test_add_field(self, mock_remove_field, mock_callable,
callable_default):
model_options = model._ModelOptions(self.__class__)
test_field = model.Field(name=mock.sentinel.name,
key=mock.sentinel.key,
is_required=False,
default=mock.sentinel.default)
mock_callable.return_value = callable_default
model_options.add_field(test_field)
mock_remove_field.assert_called_once_with(mock.sentinel.name)
self.assertIs(model_options._fields[test_field.name], test_field)
if callable_default:
self.assertIs(model_options._default_callables[test_field.key],
mock.sentinel.default)
else:
self.assertIs(model_options._defaults[test_field.key],
mock.sentinel.default)
def test_add_field(self):
self._test_add_field(callable_default=False)
self._test_add_field(callable_default=True)
@mock.patch("six.callable")
def _test_remove_field(self, mock_callable, callable_default):
mock_callable.return_value = callable_default
model_options = model._ModelOptions(self.__class__)
test_field = model.Field(name=mock.sentinel.name,
key="test_field",
is_required=False,
default=mock.sentinel.default)
model_options.add_field(test_field)
model_options.remove_field(test_field.name)
self.assertNotIn(test_field.name, model_options._fields)
if callable_default:
self.assertNotIn(test_field.name,
model_options._default_callables)
else:
self.assertNotIn(test_field.name, model_options._defaults)
def test_remove_field(self):
self._test_remove_field(callable_default=False)
self._test_remove_field(callable_default=True)
def test_get_defaults(self):
test_field = model.Field(
name=mock.sentinel.name, key=mock.sentinel.key,
is_required=False, default=lambda: mock.sentinel.default)
model_options = model._ModelOptions(self.__class__)
model_options.add_field(test_field)
defaults = model_options.get_defaults()
self.assertEqual(defaults, {mock.sentinel.key: mock.sentinel.default})
class TestBaseModel(unittest.TestCase):
def test_create_model(self):
class _Test(model.Model):
field1 = model.Field(name="field1", key="field1", default=1)
self.assertTrue(hasattr(_Test, "_meta"))
self.assertEqual(_Test().field1, 1)
def test_inherit_fields(self):
class _TestBase(model.Model):
field1 = model.Field(name="field1", key="key1",
default=1)
field2 = model.Field(name="field2", key="key2")
class _Test(_TestBase):
field2 = model.Field(name="field2", key="key2",
default=2)
class _FinalTest(_Test):
field3 = model.Field(name="field3", key="key3",
is_required=True)
final_test = _FinalTest(field3=3)
self.assertEqual(final_test.field1, 1)
self.assertEqual(final_test.field2, 2)
self.assertEqual(final_test.field3, 3)
class TestModel(unittest.TestCase):
class _Test(model.Model):
field1 = model.Field(name="field1", key="key1",
is_required=True,
is_property=False)
field2 = model.Field(name="field2", key="key2",
is_required=False,
is_property=False)
field3 = model.Field(name="field3", key="key3",
is_required=True,
is_property=True)
def setUp(self):
self._raw_data = {"key1": 1, "key2": 2, "properties": {"key3": 3}}
def test_model_eq(self):
test = self._Test(field1=1, field3=2)
test2 = self._Test(field1=1, field3=3)
self.assertFalse(test == self)
self.assertTrue(test == test)
self.assertFalse(test == test2)
def test_validate(self):
test = self._Test(field1=1, key="test_validate")
self.assertRaises(exception.DataProcessingError, test.validate)
def test_update(self):
test = self._Test(field1="change_me", field2="change_me")
test.update({"field1": mock.sentinel.field1,
"field2": mock.sentinel.field2})
self.assertIs(test.field1, mock.sentinel.field1)
self.assertIs(test.field2, mock.sentinel.field2)
def test_dump(self):
test = self._Test(field1=1, field2=2, field3=3)
self.assertEqual(test.dump(), self._raw_data)
def test_from_raw_data(self):
test = self._Test.from_raw_data(self._raw_data)
self.assertEqual(test.field1, 1)
self.assertEqual(test.field2, 2)
self.assertEqual(test.field3, 3)

3
requirements.txt
View File

@ -1,4 +1,7 @@
pbr>=1.8
six>=1.7.0
oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
oslo.log>=3.11.0 # Apache-2.0
requests
requests_ntlm