Merge pull request #1 from alexcoman/feature/data-model

Add base data model in order to improve data representation

.gitignore

@@ -0,0 +1,55 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.eggs
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+.testrepository
+.venv
+cover
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+doc/build
+
+# pbr generates these
+AUTHORS
+ChangeLog
+
+# Editors
+*~
+.*.swp
+.*sw?

.pylintrc

@@ -13,7 +13,7 @@ ignore=CVS
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
-ignore-patterns=
+ignore-patterns=test_*
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -166,12 +166,12 @@ ignore-mixin-members=yes
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=hnv_client.tests
+ignored-modules=
 
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local, hnv_client.common.model.Model
 
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
@@ -366,7 +366,7 @@ analyse-fallback-blocks=no
 [DESIGN]
 
 # Maximum number of arguments for function / method
-max-args=5
+max-args=15
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore
@@ -391,7 +391,7 @@ max-parents=7
 max-attributes=7
 
 # Minimum number of public methods for a class (see R0903).
-min-public-methods=2
+min-public-methods=1
 
 # Maximum number of public methods for a class (see R0904).
 max-public-methods=20

hnv_client/common/constant.py

@@ -0,0 +1,15 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Shared constants across the bcbio-nextgen-vm project."""

hnv_client/common/exception.py

@@ -51,6 +51,13 @@ class HNVException(Exception):
         super(HNVException, self).__init__(message)
 
 
+class DataProcessingError(HNVException):
+
+    """Base exception class for data processing related errors."""
+
+    template = "The provided information is incomplete or invalid."
+
+
 class NotFound(HNVException):
 
     """The required object is not available in container."""

hnv_client/common/model.py

@@ -0,0 +1,308 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""This module contains data models used across the project."""
+
+# pylint: disable=protected-access
+
+import copy
+
+from oslo_log import log as logging
+import six
+
+from hnv_client.common import exception
+
+LOG = logging.getLogger(__name__)
+
+
+class _FieldDescriptor(object):
+
+    """Descriptor for all the available fields for a model.
+
+    Fields are exposed as descriptors in order to control access to the
+    underlying raw data.
+    """
+
+    def __init__(self, field):
+        self._field = field
+        self._attribute = field.key
+
+    @property
+    def field(self):
+        """Expose the received field object."""
+        return self._field
+
+    def __get__(self, instance, instance_type=None):
+        if instance is not None:
+            return instance._data.get(self._attribute)
+        return self._field
+
+    def __set__(self, instance, value):
+        if instance.provision_done:
+            if self._field.is_read_only:
+                raise TypeError("%r does not support item assignment" %
+                                self._field.name)
+
+        # Update the value of the field
+        instance._data[self._attribute] = value
+        # Keep track of the changes
+        instance._changes[self._attribute] = value
+
+
+class Field(object):
+
+    """Meta information regarding the data components.
+
+    :param name:          The name of the current piece of information.
+    :param key:           The internal name for the current piece of
+                          information.
+    :param default:       Default value for the current field.
+                          (default: `None`)
+    :param is_required:   Whether the current piece of information is required
+                          for the container object or can be missing.
+                          (default: `False`)
+    :param is_property:   Whether the current piece of information is a
+                          propriety of the model. (default: `True`)
+    :param is_read_only:  Whether the current piece of information can
+                          be updated. (Default: `False`)
+    """
+
+    def __init__(self, name, key, default=None, is_required=False,
+                 is_property=True, is_read_only=False):
+        self._name = name
+        self._key = key
+        self._default = default
+        self._is_required = is_required
+        self._is_property = is_property
+        self._is_read_only = is_read_only
+
+    @property
+    def name(self):
+        """The name of the current field."""
+        return self._name
+
+    @property
+    def key(self):
+        """The internal name of the current field."""
+        return self._key
+
+    @property
+    def default(self):
+        """Default value for the current field."""
+        return self._default
+
+    @property
+    def is_required(self):
+        """Whether the current field is required or can be missing."""
+        return self._is_required
+
+    @property
+    def is_property(self):
+        """Whether the current field is a model property."""
+        return self._is_property
+
+    @property
+    def is_read_only(self):
+        """Whether the current field can be updated."""
+        return self._is_read_only
+
+    def add_to_class(self, model_class):
+        """Replace the `Field` attribute with a named `_FieldDescriptor`.
+
+        .. note::
+            This method is called  during construction of the `Model`.
+        """
+        model_class._meta.add_field(self)
+        setattr(model_class, self.name, _FieldDescriptor(self))
+
+
+class _ModelOptions(object):
+
+    """Container for all the model options.
+
+    .. note::
+        The current object will be created by the model metaclass.
+    """
+
+    def __init__(self, cls):
+        self._model_class = cls
+        self._name = cls.__name__
+
+        self._fields = {}
+        self._defaults = {}
+        self._default_callables = {}
+
+    @property
+    def fields(self):
+        """All the available fields for the current model."""
+        return self._fields
+
+    def add_field(self, field):
+        """Add the received field to the model."""
+        self.remove_field(field.name)
+        self._fields[field.name] = field
+
+        if field.default is not None:
+            if six.callable(field.default):
+                self._default_callables[field.key] = field.default
+            else:
+                self._defaults[field.key] = field.default
+
+    def remove_field(self, field_name):
+        """Remove the field with the received field name from model."""
+        field = self._fields.pop(field_name, None)
+        if field is not None and field.default is not None:
+            if six.callable(field.default):
+                self._default_callables.pop(field.name, None)
+            else:
+                self._defaults.pop(field.name, None)
+
+    def get_defaults(self):
+        """Get a dictionary that contains all the available defaults."""
+        defaults = self._defaults.copy()
+        for field_name, default in self._default_callables.items():
+            defaults[field_name] = default()
+        return defaults
+
+
+class _BaseModel(type):
+
+    """Metaclass used for properly setting up a new model."""
+
+    def __new__(mcs, name, bases, attrs):
+        # The inherit is made by deep copying the underlying field into
+        # the attributes of the new model.
+        for base in bases:
+            for key, attribute in base.__dict__.items():
+                if key not in attrs and isinstance(attribute,
+                                                   _FieldDescriptor):
+                    attrs[key] = copy.deepcopy(attribute.field)
+
+        # Initialize the new class and set the magic attributes
+        cls = super(_BaseModel, mcs).__new__(mcs, name, bases, attrs)
+
+        # Create the _ModelOptions object and inject it in the new class
+        setattr(cls, "_meta", _ModelOptions(cls))
+
+        # Get all the available fields for the current model.
+        for name, field in list(cls.__dict__.items()):
+            if isinstance(field, Field) and not name.startswith("_"):
+                field.add_to_class(cls)
+
+        # Create string representation for the current model before finalizing
+        setattr(cls, '__str__', lambda self: '%s' % cls.__name__)
+        return cls
+
+
+@six.add_metaclass(_BaseModel)
+class Model(object):
+
+    """Container for meta information regarding the data structure."""
+
+    def __init__(self, **fields):
+        self._data = self._meta.get_defaults()
+        self._changes = {}
+        self._provision_done = False
+
+        for field in self._meta.fields.values():
+            value = fields.pop(field.name, None)
+            if field.key not in self._data or value:
+                setattr(self, field.name, value)
+
+        if fields:
+            LOG.debug("Unrecognized fields: %r", fields)
+
+        self._provision_done = True
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+
+        my_properties = self.dump().get("properties", {})
+        other_properties = other.dump().get("properties", {})
+        return my_properties == other_properties
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    @classmethod
+    def from_raw_data(cls, raw_data):
+        """Create a new model using raw API response."""
+        content = {}
+        properties = raw_data.pop("properties", {})
+        for field_name, field in cls._meta.fields.items():
+            if field.is_property:
+                value = properties.pop(field.key, None)
+            else:
+                value = raw_data.pop(field.key, None)
+            content[field_name] = value
+
+        if raw_data:
+            LOG.debug("Unrecognized fields: %r", raw_data)
+        if properties:
+            LOG.debug("Unrecognized properties: %r", properties)
+
+        return cls(**content)
+
+    @property
+    def provision_done(self):
+        """Whether the creation of the model is complete."""
+        return self._provision_done
+
+    def validate(self):
+        """Check if the current model was properly created."""
+        for field_name, field in self._meta.fields.items():
+            if field.is_required and self._data.get(field.name) is None:
+                raise exception.DataProcessingError(
+                    "The required field %(field)r is missing.",
+                    field=field_name)
+
+    def update(self, fields=None):
+        """Update the value of one or more fields."""
+        for field_name, field in self._meta.fields.items():
+            if field_name in fields:
+                self._changes[field.key] = fields[field_name]
+        self._data.update(self._changes)
+
+    def commit(self, wait=False, timeout=None):
+        """Apply all the changes on the current model."""
+        # pylint: disable=unused-argument
+        self._data.update(self._changes)
+        self._changes.clear()
+
+    def dump(self, include_read_only=True):
+        """Create a dictionary with the content of the current model."""
+        content = {}
+        for field in self._meta.fields.values():
+            if field.is_read_only and not include_read_only:
+                continue
+
+            value = self._data.get(field.key)
+            if isinstance(value, Model):
+                # The raw content of the model is required
+                value = value.dump()
+
+            if not field.is_required and value is None:
+                # The value of this field is not relevant
+                continue
+
+            if field.is_property:
+                # The current field is a property and its value should
+                # be stored into the `properties` key.
+                properties = content.setdefault("properties", {})
+                properties[field.key] = value
+            else:
+                content[field.key] = value
+
+        return content

hnv_client/tests/common/test_model.py

@@ -0,0 +1,238 @@
+# Copyright 2017 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# pylint: disable=protected-access, missing-docstring
+
+import unittest
+
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+
+from hnv_client.common import exception
+from hnv_client.common import model
+
+
+class TestFieldDescriptor(unittest.TestCase):
+
+    def test_field_access(self):
+        instance = mock.Mock()
+        field = mock.Mock()
+        field_descriptor = model._FieldDescriptor(field)
+
+        self.assertIs(field, field_descriptor.__get__(None))
+
+        field_descriptor.__get__(instance)
+        instance._data.get.assert_called_once_with(field.key)
+
+    def test_set_field(self):
+        instance = mock.MagicMock()
+        instance._changes = {}
+
+        field = mock.Mock()
+        field.is_read_only = False
+        field_descriptor = model._FieldDescriptor(field)
+        field_descriptor.__set__(instance, mock.sentinel.value)
+
+        self.assertIs(instance._changes[field.key], mock.sentinel.value)
+
+    def test_set_read_only_field(self):
+        instance = mock.MagicMock()
+
+        field = mock.Mock()
+        field.is_read_only = True
+        field_descriptor = model._FieldDescriptor(field)
+        with self.assertRaises(TypeError):
+            field_descriptor.__set__(instance, mock.sentinel.value)
+
+    def test_field_property(self):
+        field = mock.Mock()
+        field_descriptor = model._FieldDescriptor(field)
+
+        self.assertIs(field_descriptor.field, field)
+
+
+class TestField(unittest.TestCase):
+
+    def test_properties(self):
+        field = model.Field(name=mock.sentinel.name,
+                            key=mock.sentinel.name,
+                            default=mock.sentinel.default,
+                            is_required=mock.sentinel.is_required,
+                            is_property=mock.sentinel.is_property,
+                            is_read_only=mock.sentinel.is_read_only)
+
+        self.assertIs(field.name, mock.sentinel.name)
+        self.assertIs(field.default, mock.sentinel.default)
+        self.assertIs(field.is_required, mock.sentinel.is_required)
+        self.assertIs(field.is_property, mock.sentinel.is_property)
+        self.assertIs(field.is_read_only, mock.sentinel.is_read_only)
+
+    @mock.patch("hnv_client.common.model._FieldDescriptor")
+    def test_add_to_class(self, mock_field_descriptor):
+        field = model.Field(name="test_add_to_class", key="test")
+        model_class = mock.Mock()
+
+        field.add_to_class(model_class)
+
+        mock_field_descriptor.assert_called_once_with(field)
+        self.assertIsNotNone(getattr(model_class, "test_add_to_class"))
+
+
+class TestModelOptions(unittest.TestCase):
+
+    def test_initialization(self):
+        mock.sentinel.cls.__name__ = mock.sentinel.cls.name
+        model_options = model._ModelOptions(cls=mock.sentinel.cls)
+
+        self.assertIs(model_options._model_class, mock.sentinel.cls)
+        self.assertEqual(model_options._name, mock.sentinel.cls.name)
+
+    @mock.patch("six.callable")
+    @mock.patch("hnv_client.common.model._ModelOptions.remove_field")
+    def _test_add_field(self, mock_remove_field, mock_callable,
+                        callable_default):
+        model_options = model._ModelOptions(self.__class__)
+        test_field = model.Field(name=mock.sentinel.name,
+                                 key=mock.sentinel.key,
+                                 is_required=False,
+                                 default=mock.sentinel.default)
+        mock_callable.return_value = callable_default
+
+        model_options.add_field(test_field)
+
+        mock_remove_field.assert_called_once_with(mock.sentinel.name)
+        self.assertIs(model_options._fields[test_field.name], test_field)
+        if callable_default:
+            self.assertIs(model_options._default_callables[test_field.key],
+                          mock.sentinel.default)
+        else:
+            self.assertIs(model_options._defaults[test_field.key],
+                          mock.sentinel.default)
+
+    def test_add_field(self):
+        self._test_add_field(callable_default=False)
+        self._test_add_field(callable_default=True)
+
+    @mock.patch("six.callable")
+    def _test_remove_field(self, mock_callable, callable_default):
+        mock_callable.return_value = callable_default
+        model_options = model._ModelOptions(self.__class__)
+        test_field = model.Field(name=mock.sentinel.name,
+                                 key="test_field",
+                                 is_required=False,
+                                 default=mock.sentinel.default)
+        model_options.add_field(test_field)
+
+        model_options.remove_field(test_field.name)
+
+        self.assertNotIn(test_field.name, model_options._fields)
+        if callable_default:
+            self.assertNotIn(test_field.name,
+                             model_options._default_callables)
+        else:
+            self.assertNotIn(test_field.name, model_options._defaults)
+
+    def test_remove_field(self):
+        self._test_remove_field(callable_default=False)
+        self._test_remove_field(callable_default=True)
+
+    def test_get_defaults(self):
+        test_field = model.Field(
+            name=mock.sentinel.name, key=mock.sentinel.key,
+            is_required=False, default=lambda: mock.sentinel.default)
+        model_options = model._ModelOptions(self.__class__)
+        model_options.add_field(test_field)
+
+        defaults = model_options.get_defaults()
+
+        self.assertEqual(defaults, {mock.sentinel.key: mock.sentinel.default})
+
+
+class TestBaseModel(unittest.TestCase):
+
+    def test_create_model(self):
+
+        class _Test(model.Model):
+            field1 = model.Field(name="field1", key="field1", default=1)
+
+        self.assertTrue(hasattr(_Test, "_meta"))
+        self.assertEqual(_Test().field1, 1)
+
+    def test_inherit_fields(self):
+
+        class _TestBase(model.Model):
+            field1 = model.Field(name="field1", key="key1",
+                                 default=1)
+            field2 = model.Field(name="field2", key="key2")
+
+        class _Test(_TestBase):
+            field2 = model.Field(name="field2", key="key2",
+                                 default=2)
+
+        class _FinalTest(_Test):
+            field3 = model.Field(name="field3", key="key3",
+                                 is_required=True)
+
+        final_test = _FinalTest(field3=3)
+        self.assertEqual(final_test.field1, 1)
+        self.assertEqual(final_test.field2, 2)
+        self.assertEqual(final_test.field3, 3)
+
+
+class TestModel(unittest.TestCase):
+
+    class _Test(model.Model):
+        field1 = model.Field(name="field1", key="key1",
+                             is_required=True,
+                             is_property=False)
+        field2 = model.Field(name="field2", key="key2",
+                             is_required=False,
+                             is_property=False)
+        field3 = model.Field(name="field3", key="key3",
+                             is_required=True,
+                             is_property=True)
+
+    def setUp(self):
+        self._raw_data = {"key1": 1, "key2": 2, "properties": {"key3": 3}}
+
+    def test_model_eq(self):
+        test = self._Test(field1=1, field3=2)
+        test2 = self._Test(field1=1, field3=3)
+        self.assertFalse(test == self)
+        self.assertTrue(test == test)
+        self.assertFalse(test == test2)
+
+    def test_validate(self):
+        test = self._Test(field1=1, key="test_validate")
+        self.assertRaises(exception.DataProcessingError, test.validate)
+
+    def test_update(self):
+        test = self._Test(field1="change_me", field2="change_me")
+        test.update({"field1": mock.sentinel.field1,
+                     "field2": mock.sentinel.field2})
+
+        self.assertIs(test.field1, mock.sentinel.field1)
+        self.assertIs(test.field2, mock.sentinel.field2)
+
+    def test_dump(self):
+        test = self._Test(field1=1, field2=2, field3=3)
+        self.assertEqual(test.dump(), self._raw_data)
+
+    def test_from_raw_data(self):
+        test = self._Test.from_raw_data(self._raw_data)
+        self.assertEqual(test.field1, 1)
+        self.assertEqual(test.field2, 2)
+        self.assertEqual(test.field3, 3)

requirements.txt

@@ -1,4 +1,7 @@
 pbr>=1.8
 six>=1.7.0
+oslo.config!=3.18.0,>=3.14.0 # Apache-2.0
+oslo.i18n>=2.1.0 # Apache-2.0
+oslo.log>=3.11.0 # Apache-2.0
 requests
 requests_ntlm