From 6251ca534426a58fe557d89160d5a60a2933efb0 Mon Sep 17 00:00:00 2001
From: Alexandru Coman <acoman@cloudbasesolutions.com>
Date: Mon, 9 Jan 2017 16:02:03 +0200
Subject: [PATCH] Add base data model in order to improve data representation
The proposed data model improves the way how the information
is structured and allows related entities to to inherit data.
---
.gitignore | 55 +++++
.pylintrc | 10 +-
hnv_client/common/constant.py | 15 ++
hnv_client/common/exception.py | 7 +
hnv_client/common/model.py | 308 ++++++++++++++++++++++++++
hnv_client/tests/common/__init__.py | 0
hnv_client/tests/common/test_model.py | 238 ++++++++++++++++++++
requirements.txt | 3 +
8 files changed, 631 insertions(+), 5 deletions(-)
create mode 100644 .gitignore
create mode 100644 hnv_client/common/model.py
create mode 100644 hnv_client/tests/common/__init__.py
create mode 100644 hnv_client/tests/common/test_model.py
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f3f82ad
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,55 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.eggs
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+.testrepository
+.venv
+cover
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+doc/build
+
+# pbr generates these
+AUTHORS
+ChangeLog
+
+# Editors
+*~
+.*.swp
+.*sw?
diff --git a/.pylintrc b/.pylintrc
index b49244c..a73c8b3 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -13,7 +13,7 @@ ignore=CVS
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.
-ignore-patterns=
+ignore-patterns=test_*
# Pickle collected data for later comparisons.
persistent=yes
@@ -166,12 +166,12 @@ ignore-mixin-members=yes
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=hnv_client.tests
+ignored-modules=
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local, hnv_client.common.model.Model
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
@@ -366,7 +366,7 @@ analyse-fallback-blocks=no
[DESIGN]
# Maximum number of arguments for function / method
-max-args=5
+max-args=15
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
@@ -391,7 +391,7 @@ max-parents=7
max-attributes=7
# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
+min-public-methods=1
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
diff --git a/hnv_client/common/constant.py b/hnv_client/common/constant.py
index e69de29..c4dd331 100644
--- a/hnv_client/common/constant.py
+++ b/hnv_client/common/constant.py
@@ -0,0 +1,15 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Shared constants across the bcbio-nextgen-vm project."""
diff --git a/hnv_client/common/exception.py b/hnv_client/common/exception.py
index c13a68a..fc89e27 100644
--- a/hnv_client/common/exception.py
+++ b/hnv_client/common/exception.py
@@ -51,6 +51,13 @@ class HNVException(Exception):
super(HNVException, self).__init__(message)
+class DataProcessingError(HNVException):
+
+ """Base exception class for data processing related errors."""
+
+ template = "The provided information is incomplete or invalid."
+
+
class NotFound(HNVException):
"""The required object is not available in container."""
diff --git a/hnv_client/common/model.py b/hnv_client/common/model.py
new file mode 100644
index 0000000..02e0a6b
--- /dev/null
+++ b/hnv_client/common/model.py
@@ -0,0 +1,308 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""This module contains data models used across the project."""
+
+# pylint: disable=protected-access
+
+import copy
+
+from oslo_log import log as logging
+import six
+
+from hnv_client.common import exception
+
+LOG = logging.getLogger(__name__)
+
+
+class _FieldDescriptor(object):
+
+ """Descriptor for all the available fields for a model.
+
+ Fields are exposed as descriptors in order to control access to the
+ underlying raw data.
+ """
+
+ def __init__(self, field):
+ self._field = field
+ self._attribute = field.key
+
+ @property
+ def field(self):
+ """Expose the received field object."""
+ return self._field
+
+ def __get__(self, instance, instance_type=None):
+ if instance is not None:
+ return instance._data.get(self._attribute)
+ return self._field
+
+ def __set__(self, instance, value):
+ if instance.provision_done:
+ if self._field.is_read_only:
+ raise TypeError("%r does not support item assignment" %
+ self._field.name)
+
+ # Update the value of the field
+ instance._data[self._attribute] = value
+ # Keep track of the changes
+ instance._changes[self._attribute] = value
+
+
+class Field(object):
+
+ """Meta information regarding the data components.
+
+ :param name: The name of the current piece of information.
+ :param key: The internal name for the current piece of
+ information.
+ :param default: Default value for the current field.
+ (default: `None`)
+ :param is_required: Whether the current piece of information is required
+ for the container object or can be missing.
+ (default: `False`)
+ :param is_property: Whether the current piece of information is a
+ propriety of the model. (default: `True`)
+ :param is_read_only: Whether the current piece of information can
+ be updated. (Default: `False`)
+ """
+
+ def __init__(self, name, key, default=None, is_required=False,
+ is_property=True, is_read_only=False):
+ self._name = name
+ self._key = key
+ self._default = default
+ self._is_required = is_required
+ self._is_property = is_property
+ self._is_read_only = is_read_only
+
+ @property
+ def name(self):
+ """The name of the current field."""
+ return self._name
+
+ @property
+ def key(self):
+ """The internal name of the current field."""
+ return self._key
+
+ @property
+ def default(self):
+ """Default value for the current field."""
+ return self._default
+
+ @property
+ def is_required(self):
+ """Whether the current field is required or can be missing."""
+ return self._is_required
+
+ @property
+ def is_property(self):
+ """Whether the current field is a model property."""
+ return self._is_property
+
+ @property
+ def is_read_only(self):
+ """Whether the current field can be updated."""
+ return self._is_read_only
+
+ def add_to_class(self, model_class):
+ """Replace the `Field` attribute with a named `_FieldDescriptor`.
+
+ .. note::
+ This method is called during construction of the `Model`.
+ """
+ model_class._meta.add_field(self)
+ setattr(model_class, self.name, _FieldDescriptor(self))
+
+
+class _ModelOptions(object):
+
+ """Container for all the model options.
+
+ .. note::
+ The current object will be created by the model metaclass.
+ """
+
+ def __init__(self, cls):
+ self._model_class = cls
+ self._name = cls.__name__
+
+ self._fields = {}
+ self._defaults = {}
+ self._default_callables = {}
+
+ @property
+ def fields(self):
+ """All the available fields for the current model."""
+ return self._fields
+
+ def add_field(self, field):
+ """Add the received field to the model."""
+ self.remove_field(field.name)
+ self._fields[field.name] = field
+
+ if field.default is not None:
+ if six.callable(field.default):
+ self._default_callables[field.key] = field.default
+ else:
+ self._defaults[field.key] = field.default
+
+ def remove_field(self, field_name):
+ """Remove the field with the received field name from model."""
+ field = self._fields.pop(field_name, None)
+ if field is not None and field.default is not None:
+ if six.callable(field.default):
+ self._default_callables.pop(field.name, None)
+ else:
+ self._defaults.pop(field.name, None)
+
+ def get_defaults(self):
+ """Get a dictionary that contains all the available defaults."""
+ defaults = self._defaults.copy()
+ for field_name, default in self._default_callables.items():
+ defaults[field_name] = default()
+ return defaults
+
+
+class _BaseModel(type):
+
+ """Metaclass used for properly setting up a new model."""
+
+ def __new__(mcs, name, bases, attrs):
+ # The inherit is made by deep copying the underlying field into
+ # the attributes of the new model.
+ for base in bases:
+ for key, attribute in base.__dict__.items():
+ if key not in attrs and isinstance(attribute,
+ _FieldDescriptor):
+ attrs[key] = copy.deepcopy(attribute.field)
+
+ # Initialize the new class and set the magic attributes
+ cls = super(_BaseModel, mcs).__new__(mcs, name, bases, attrs)
+
+ # Create the _ModelOptions object and inject it in the new class
+ setattr(cls, "_meta", _ModelOptions(cls))
+
+ # Get all the available fields for the current model.
+ for name, field in list(cls.__dict__.items()):
+ if isinstance(field, Field) and not name.startswith("_"):
+ field.add_to_class(cls)
+
+ # Create string representation for the current model before finalizing
+ setattr(cls, '__str__', lambda self: '%s' % cls.__name__)
+ return cls
+
+
+@six.add_metaclass(_BaseModel)
+class Model(object):
+
+ """Container for meta information regarding the data structure."""
+
+ def __init__(self, **fields):
+ self._data = self._meta.get_defaults()
+ self._changes = {}
+ self._provision_done = False
+
+ for field in self._meta.fields.values():
+ value = fields.pop(field.name, None)
+ if field.key not in self._data or value:
+ setattr(self, field.name, value)
+
+ if fields:
+ LOG.debug("Unrecognized fields: %r", fields)
+
+ self._provision_done = True
+
+ def __eq__(self, other):
+ if not isinstance(other, self.__class__):
+ return False
+
+ my_properties = self.dump().get("properties", {})
+ other_properties = other.dump().get("properties", {})
+ return my_properties == other_properties
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ @classmethod
+ def from_raw_data(cls, raw_data):
+ """Create a new model using raw API response."""
+ content = {}
+ properties = raw_data.pop("properties", {})
+ for field_name, field in cls._meta.fields.items():
+ if field.is_property:
+ value = properties.pop(field.key, None)
+ else:
+ value = raw_data.pop(field.key, None)
+ content[field_name] = value
+
+ if raw_data:
+ LOG.debug("Unrecognized fields: %r", raw_data)
+ if properties:
+ LOG.debug("Unrecognized properties: %r", properties)
+
+ return cls(**content)
+
+ @property
+ def provision_done(self):
+ """Whether the creation of the model is complete."""
+ return self._provision_done
+
+ def validate(self):
+ """Check if the current model was properly created."""
+ for field_name, field in self._meta.fields.items():
+ if field.is_required and self._data.get(field.name) is None:
+ raise exception.DataProcessingError(
+ "The required field %(field)r is missing.",
+ field=field_name)
+
+ def update(self, fields=None):
+ """Update the value of one or more fields."""
+ for field_name, field in self._meta.fields.items():
+ if field_name in fields:
+ self._changes[field.key] = fields[field_name]
+ self._data.update(self._changes)
+
+ def commit(self, wait=False, timeout=None):
+ """Apply all the changes on the current model."""
+ # pylint: disable=unused-argument
+ self._data.update(self._changes)
+ self._changes.clear()
+
+ def dump(self, include_read_only=True):
+ """Create a dictionary with the content of the current model."""
+ content = {}
+ for field in self._meta.fields.values():
+ if field.is_read_only and not include_read_only:
+ continue
+
+ value = self._data.get(field.key)
+ if isinstance(value, Model):
+ # The raw content of the model is required
+ value = value.dump()
+
+ if not field.is_required and value is None:
+ # The value of this field is not relevant
+ continue
+
+ if field.is_property:
+ # The current field is a property and its value should
+ # be stored into the `properties` key.
+ properties = content.setdefault("properties", {})
+ properties[field.key] = value
+ else:
+ content[field.key] = value
+
+ return content
diff --git a/hnv_client/tests/common/__init__.py b/hnv_client/tests/common/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/hnv_client/tests/common/test_model.py b/hnv_client/tests/common/test_model.py
new file mode 100644
index 0000000..be33129
--- /dev/null
+++ b/hnv_client/tests/common/test_model.py
@@ -0,0 +1,238 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=protected-access, missing-docstring
+
+import unittest
+
+try:
+ import unittest.mock as mock
+except ImportError:
+ import mock
+
+from hnv_client.common import exception
+from hnv_client.common import model
+
+
+class TestFieldDescriptor(unittest.TestCase):
+
+ def test_field_access(self):
+ instance = mock.Mock()
+ field = mock.Mock()
+ field_descriptor = model._FieldDescriptor(field)
+
+ self.assertIs(field, field_descriptor.__get__(None))
+
+ field_descriptor.__get__(instance)
+ instance._data.get.assert_called_once_with(field.key)
+
+ def test_set_field(self):
+ instance = mock.MagicMock()
+ instance._changes = {}
+
+ field = mock.Mock()
+ field.is_read_only = False
+ field_descriptor = model._FieldDescriptor(field)
+ field_descriptor.__set__(instance, mock.sentinel.value)
+
+ self.assertIs(instance._changes[field.key], mock.sentinel.value)
+
+ def test_set_read_only_field(self):
+ instance = mock.MagicMock()
+
+ field = mock.Mock()
+ field.is_read_only = True
+ field_descriptor = model._FieldDescriptor(field)
+ with self.assertRaises(TypeError):
+ field_descriptor.__set__(instance, mock.sentinel.value)
+
+ def test_field_property(self):
+ field = mock.Mock()
+ field_descriptor = model._FieldDescriptor(field)
+
+ self.assertIs(field_descriptor.field, field)
+
+
+class TestField(unittest.TestCase):
+
+ def test_properties(self):
+ field = model.Field(name=mock.sentinel.name,
+ key=mock.sentinel.name,
+ default=mock.sentinel.default,
+ is_required=mock.sentinel.is_required,
+ is_property=mock.sentinel.is_property,
+ is_read_only=mock.sentinel.is_read_only)
+
+ self.assertIs(field.name, mock.sentinel.name)
+ self.assertIs(field.default, mock.sentinel.default)
+ self.assertIs(field.is_required, mock.sentinel.is_required)
+ self.assertIs(field.is_property, mock.sentinel.is_property)
+ self.assertIs(field.is_read_only, mock.sentinel.is_read_only)
+
+ @mock.patch("hnv_client.common.model._FieldDescriptor")
+ def test_add_to_class(self, mock_field_descriptor):
+ field = model.Field(name="test_add_to_class", key="test")
+ model_class = mock.Mock()
+
+ field.add_to_class(model_class)
+
+ mock_field_descriptor.assert_called_once_with(field)
+ self.assertIsNotNone(getattr(model_class, "test_add_to_class"))
+
+
+class TestModelOptions(unittest.TestCase):
+
+ def test_initialization(self):
+ mock.sentinel.cls.__name__ = mock.sentinel.cls.name
+ model_options = model._ModelOptions(cls=mock.sentinel.cls)
+
+ self.assertIs(model_options._model_class, mock.sentinel.cls)
+ self.assertEqual(model_options._name, mock.sentinel.cls.name)
+
+ @mock.patch("six.callable")
+ @mock.patch("hnv_client.common.model._ModelOptions.remove_field")
+ def _test_add_field(self, mock_remove_field, mock_callable,
+ callable_default):
+ model_options = model._ModelOptions(self.__class__)
+ test_field = model.Field(name=mock.sentinel.name,
+ key=mock.sentinel.key,
+ is_required=False,
+ default=mock.sentinel.default)
+ mock_callable.return_value = callable_default
+
+ model_options.add_field(test_field)
+
+ mock_remove_field.assert_called_once_with(mock.sentinel.name)
+ self.assertIs(model_options._fields[test_field.name], test_field)
+ if callable_default:
+ self.assertIs(model_options._default_callables[test_field.key],
+ mock.sentinel.default)
+ else:
+ self.assertIs(model_options._defaults[test_field.key],
+ mock.sentinel.default)
+
+ def test_add_field(self):
+ self._test_add_field(callable_default=False)
+ self._test_add_field(callable_default=True)
+
+ @mock.patch("six.callable")
+ def _test_remove_field(self, mock_callable, callable_default):
+ mock_callable.return_value = callable_default
+ model_options = model._ModelOptions(self.__class__)
+ test_field = model.Field(name=mock.sentinel.name,
+ key="test_field",
+ is_required=False,
+ default=mock.sentinel.default)
+ model_options.add_field(test_field)
+
+ model_options.remove_field(test_field.name)
+
+ self.assertNotIn(test_field.name, model_options._fields)
+ if callable_default:
+ self.assertNotIn(test_field.name,
+ model_options._default_callables)
+ else:
+ self.assertNotIn(test_field.name, model_options._defaults)
+
+ def test_remove_field(self):
+ self._test_remove_field(callable_default=False)
+ self._test_remove_field(callable_default=True)
+
+ def test_get_defaults(self):
+ test_field = model.Field(
+ name=mock.sentinel.name, key=mock.sentinel.key,
+ is_required=False, default=lambda: mock.sentinel.default)
+ model_options = model._ModelOptions(self.__class__)
+ model_options.add_field(test_field)
+
+ defaults = model_options.get_defaults()
+
+ self.assertEqual(defaults, {mock.sentinel.key: mock.sentinel.default})
+
+
+class TestBaseModel(unittest.TestCase):
+
+ def test_create_model(self):
+
+ class _Test(model.Model):
+ field1 = model.Field(name="field1", key="field1", default=1)
+
+ self.assertTrue(hasattr(_Test, "_meta"))
+ self.assertEqual(_Test().field1, 1)
+
+ def test_inherit_fields(self):
+
+ class _TestBase(model.Model):
+ field1 = model.Field(name="field1", key="key1",
+ default=1)
+ field2 = model.Field(name="field2", key="key2")
+
+ class _Test(_TestBase):
+ field2 = model.Field(name="field2", key="key2",
+ default=2)
+
+ class _FinalTest(_Test):
+ field3 = model.Field(name="field3", key="key3",
+ is_required=True)
+
+ final_test = _FinalTest(field3=3)
+ self.assertEqual(final_test.field1, 1)
+ self.assertEqual(final_test.field2, 2)
+ self.assertEqual(final_test.field3, 3)
+
+
+class TestModel(unittest.TestCase):
+
+ class _Test(model.Model):
+ field1 = model.Field(name="field1", key="key1",
+ is_required=True,
+ is_property=False)
+ field2 = model.Field(name="field2", key="key2",
+ is_required=False,
+ is_property=False)
+ field3 = model.Field(name="field3", key="key3",
+ is_required=True,
+ is_property=True)
+
+ def setUp(self):
+ self._raw_data = {"key1": 1, "key2": 2, "properties": {"key3": 3}}
+
+ def test_model_eq(self):
+ test = self._Test(field1=1, field3=2)
+ test2 = self._Test(field1=1, field3=3)
+ self.assertFalse(test == self)
+ self.assertTrue(test == test)
+ self.assertFalse(test == test2)
+
+ def test_validate(self):
+ test = self._Test(field1=1, key="test_validate")
+ self.assertRaises(exception.DataProcessingError, test.validate)
+
+ def test_update(self):
+ test = self._Test(field1="change_me", field2="change_me")
+ test.update({"field1": mock.sentinel.field1,
+ "field2": mock.sentinel.field2})
+
+ self.assertIs(test.field1, mock.sentinel.field1)
+ self.assertIs(test.field2, mock.sentinel.field2)
+
+ def test_dump(self):
+ test = self._Test(field1=1, field2=2, field3=3)
+ self.assertEqual(test.dump(), self._raw_data)
+
+ def test_from_raw_data(self):
+ test = self._Test.from_raw_data(self._raw_data)
+ self.assertEqual(test.field1, 1)
+ self.assertEqual(test.field2, 2)
+ self.assertEqual(test.field3, 3)
diff --git a/requirements.txt b/requirements.txt
index d7ab7fb..141ddae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,7 @@
pbr>=1.8
six>=1.7.0
+oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
+oslo.i18n>=2.1.0 # Apache-2.0
+oslo.log>=3.11.0 # Apache-2.0
requests
requests_ntlm