diff --git a/.zuul.yaml b/.zuul.yaml index f70f3e9..17d0c86 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -19,15 +19,24 @@ parent: vexxhost-promote-docker-image vars: *atmosphere_images +- job: + name: atmosphere:linters:tox + parent: tox-linters + vars: + python_version: 3.7 + - project: check: jobs: + - atmosphere:linters:tox - tox-py37 - atmosphere:image:build gate: jobs: + - atmosphere:linters:tox - tox-py37 - atmosphere:image:upload promote: jobs: + - atmosphere:linters:tox - atmosphere:image:promote diff --git a/atmosphere/api/ingress.py b/atmosphere/api/ingress.py index 2cd3656..a369d84 100644 --- a/atmosphere/api/ingress.py +++ b/atmosphere/api/ingress.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Ingress + +""" + from flask import Blueprint from flask import request from flask import abort from flask import jsonify -from dateutil.relativedelta import relativedelta from atmosphere.app import create_app from atmosphere import exceptions @@ -27,6 +30,7 @@ blueprint = Blueprint('ingress', __name__) def init_application(config=None): + """init_application""" app = create_app(config) app.register_blueprint(blueprint) return app @@ -34,20 +38,17 @@ def init_application(config=None): @blueprint.route('/v1/event', methods=['POST']) def event(): + """event""" if request.json is None: abort(400) - for event in request.json: - print(jsonify(event).get_data(True)) - event = utils.normalize_event(event) + for event_data in request.json: + print(jsonify(event_data).get_data(True)) + event_data = utils.normalize_event(event_data) try: - resource = models.Resource.get_or_create(event) + models.Resource.get_or_create(event_data) except (exceptions.EventTooOld, exceptions.IgnoredEvent): return '', 202 - # TODO(mnaser): Drop this logging eventually... - print(jsonify(event).get_data(True)) - print(jsonify(resource.serialize).get_data(True)) - return '', 204 diff --git a/atmosphere/app.py b/atmosphere/app.py index 88487f8..8c69c2c 100644 --- a/atmosphere/app.py +++ b/atmosphere/app.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""App + +""" import os from flask import Flask @@ -20,6 +23,7 @@ from atmosphere import models def create_app(config=None): + """create_app""" app = Flask(__name__) if config is not None: @@ -39,5 +43,3 @@ def create_app(config=None): models.migrate.init_app(app, models.db, directory=migrations_path) return app - - diff --git a/atmosphere/exceptions.py b/atmosphere/exceptions.py index 8e1f7fb..1052066 100644 --- a/atmosphere/exceptions.py +++ b/atmosphere/exceptions.py @@ -12,20 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Exceptions + +""" + from werkzeug import exceptions class UnsupportedEventType(exceptions.BadRequest): + """UnsupportedEventType""" description = 'Unsupported event type' class MultipleOpenPeriods(exceptions.Conflict): + """MultipleOpenPeriods""" description = 'Multiple open periods' class IgnoredEvent(Exception): + """IgnoredEvent""" description = 'Ignored event type' class EventTooOld(Exception): - pass + """EventTooOld""" diff --git a/atmosphere/models.py b/atmosphere/models.py index f0c9f06..a96ef81 100644 --- a/atmosphere/models.py +++ b/atmosphere/models.py @@ -12,18 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Models + +""" +# pylint: disable=R0903 +# pylint: disable=W0223 +# pylint: disable=no-member +# pylint: disable=not-an-iterable from datetime import datetime +from dateutil.relativedelta import relativedelta from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate -from sqlalchemy import func from sqlalchemy import exc from sqlalchemy.orm import exc as orm_exc -from dateutil.relativedelta import relativedelta from sqlalchemy.types import TypeDecorator from atmosphere import exceptions -from atmosphere import utils db = SQLAlchemy() migrate = Migrate() @@ -32,11 +37,44 @@ migrate = Migrate() MONTH_START = relativedelta(day=1, hour=0, minute=0, second=0, microsecond=0) +def get_model_type_from_event(event): + """get_model_type_from_event""" + if event.startswith('compute.instance'): + return Instance, InstanceSpec + if event.startswith('aggregate.'): + raise exceptions.IgnoredEvent + if event.startswith('compute_task.'): + raise exceptions.IgnoredEvent + if event.startswith('compute.'): + raise exceptions.IgnoredEvent + if event.startswith('flavor.'): + raise exceptions.IgnoredEvent + if event.startswith('keypair.'): + raise exceptions.IgnoredEvent + if event.startswith('libvirt.'): + raise exceptions.IgnoredEvent + if event.startswith('metrics.'): + raise exceptions.IgnoredEvent + if event.startswith('scheduler.'): + raise exceptions.IgnoredEvent + if event.startswith('server_group.'): + raise exceptions.IgnoredEvent + if event.startswith('service.'): + raise exceptions.IgnoredEvent + if event == 'volume.usage': + raise exceptions.IgnoredEvent + + raise exceptions.UnsupportedEventType + + class GetOrCreateMixin: + """GetOrCreateMixin""" + @classmethod - def get_or_create(self, event): - query = self.query_from_event(event) - new_instance = self.from_event(event) + def get_or_create(cls, event): + """get_or_create""" + query = cls.query_from_event(event) + new_instance = cls.from_event(event) db_instance = query.first() if db_instance is None: @@ -54,6 +92,8 @@ class GetOrCreateMixin: class Resource(db.Model, GetOrCreateMixin): + """Resource""" + uuid = db.Column(db.String(36), primary_key=True) type = db.Column(db.String(32), nullable=False) project = db.Column(db.String(32), nullable=False) @@ -66,8 +106,9 @@ class Resource(db.Model, GetOrCreateMixin): } @classmethod - def from_event(self, event): - cls, _ = utils.get_model_type_from_event(event['event_type']) + def from_event(cls, event): + """from_event""" + cls, _ = get_model_type_from_event(event['event_type']) return cls( uuid=event['traits']['resource_id'], @@ -76,8 +117,9 @@ class Resource(db.Model, GetOrCreateMixin): ) @classmethod - def query_from_event(self, event): - cls, _ = utils.get_model_type_from_event(event['event_type']) + def query_from_event(cls, event): + """query_from_event""" + cls, _ = get_model_type_from_event(event['event_type']) return cls.query.filter_by( uuid=event['traits']['resource_id'], @@ -85,8 +127,9 @@ class Resource(db.Model, GetOrCreateMixin): ).with_for_update() @classmethod - def get_or_create(self, event): - resource = super(Resource, self).get_or_create(event) + def get_or_create(cls, event): + """get_or_create""" + resource = super(Resource, cls).get_or_create(event) # If the last update is newer than our last update, we assume that # another event has been processed that is newer (so we should ignore @@ -137,6 +180,7 @@ class Resource(db.Model, GetOrCreateMixin): return resource def get_open_period(self): + """get_open_period""" open_periods = list(filter(lambda p: p.ended_at is None, self.periods)) if len(open_periods) > 1: raise exceptions.MultipleOpenPeriods @@ -154,16 +198,19 @@ class Resource(db.Model, GetOrCreateMixin): 'project': self.project, 'updated_at': self.updated_at, 'periods': [p.serialize for p in self.periods], - } + } class Instance(Resource): + """Instance""" + __mapper_args__ = { 'polymorphic_identity': 'OS::Nova::Server' } @classmethod - def is_event_ignored(self, event): + def is_event_ignored(cls, event): + """is_event_ignored""" vm_state_is_deleted = (event['traits']['state'] == 'deleted') no_deleted_at = ('deleted_at' not in event['traits']) @@ -174,21 +221,27 @@ class Instance(Resource): class BigIntegerDateTime(TypeDecorator): + """BigIntegerDateTime""" + impl = db.BigInteger def process_bind_param(self, value, _): + """process_bind_param""" if value is None: return None assert isinstance(value, datetime) return value.timestamp() * 1000 def process_result_value(self, value, _): + """process_result_value""" if value is None: return None return datetime.fromtimestamp(value / 1000) class Period(db.Model): + """Period""" + id = db.Column(db.Integer, primary_key=True) resource_uuid = db.Column(db.String(36), db.ForeignKey('resource.uuid'), nullable=False) @@ -200,6 +253,7 @@ class Period(db.Model): @property def seconds(self): + """seconds""" ended_at = self.ended_at if ended_at is None: ended_at = datetime.now() @@ -214,10 +268,12 @@ class Period(db.Model): 'ended_at': self.ended_at, 'seconds': self.seconds, 'spec': self.spec.serialize, - } + } class Spec(db.Model, GetOrCreateMixin): + """Spec""" + id = db.Column(db.Integer, primary_key=True) type = db.Column(db.String(32)) @@ -226,16 +282,18 @@ class Spec(db.Model, GetOrCreateMixin): } @classmethod - def from_event(self, event): - _, cls = utils.get_model_type_from_event(event['event_type']) + def from_event(cls, event): + """from_event""" + _, cls = get_model_type_from_event(event['event_type']) spec = {c.name: event['traits'][c.name] for c in cls.__table__.columns if c.name != 'id'} return cls(**spec) @classmethod - def query_from_event(self, event): - _, cls = utils.get_model_type_from_event(event['event_type']) + def query_from_event(cls, event): + """query_from_event""" + _, cls = get_model_type_from_event(event['event_type']) spec = {c.name: event['traits'][c.name] for c in cls.__table__.columns if c.name != 'id'} @@ -243,6 +301,8 @@ class Spec(db.Model, GetOrCreateMixin): class InstanceSpec(Spec): + """InstanceSpec""" + id = db.Column(db.Integer, db.ForeignKey('spec.id'), primary_key=True) instance_type = db.Column(db.String(255)) state = db.Column(db.String(255)) @@ -252,7 +312,7 @@ class InstanceSpec(Spec): ) __mapper_args__ = { - 'polymorphic_identity': 'OS::Nova::Server', + 'polymorphic_identity': 'OS::Nova::Server', } @property @@ -262,4 +322,4 @@ class InstanceSpec(Spec): return { 'instance_type': self.instance_type, 'state': self.state, - } + } diff --git a/atmosphere/tests/unit/test_utils.py b/atmosphere/tests/unit/test_utils.py index 4af1d4a..99b180d 100644 --- a/atmosphere/tests/unit/test_utils.py +++ b/atmosphere/tests/unit/test_utils.py @@ -44,18 +44,18 @@ class TestNormalizeEvent: class TestModelTypeDetection: def test_compute_instance(self): - assert utils.get_model_type_from_event('compute.instance.exists') == \ + assert models.get_model_type_from_event('compute.instance.exists') == \ (models.Instance, models.InstanceSpec) def test_ignored_resource(self, ignored_event): with pytest.raises(exceptions.IgnoredEvent) as e: - utils.get_model_type_from_event(ignored_event) + models.get_model_type_from_event(ignored_event) assert e.value.description == "Ignored event type" def test_unknown_resource(self): with pytest.raises(exceptions.UnsupportedEventType) as e: - utils.get_model_type_from_event('foobar') + models.get_model_type_from_event('foobar') assert e.value.code == 400 assert e.value.description == "Unsupported event type" diff --git a/atmosphere/utils.py b/atmosphere/utils.py index a8d5d74..cfc6cb7 100644 --- a/atmosphere/utils.py +++ b/atmosphere/utils.py @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Utils + +""" + from ceilometer.event import models as ceilometer_models from dateutil import parser -from atmosphere import exceptions -from atmosphere import models - def normalize_event(event): + """normalize_event""" event['generated'] = parser.parse(event['generated']) event['traits'] = { k: ceilometer_models.Trait.convert_value(t, v) @@ -27,34 +29,3 @@ def normalize_event(event): } return event - - -def get_model_type_from_event(event): - if event.startswith('compute.instance'): - return models.Instance, models.InstanceSpec - if event.startswith('aggregate.'): - raise exceptions.IgnoredEvent - if event.startswith('compute_task.'): - raise exceptions.IgnoredEvent - if event.startswith('compute.'): - raise exceptions.IgnoredEvent - if event.startswith('flavor.'): - raise exceptions.IgnoredEvent - if event.startswith('keypair.'): - raise exceptions.IgnoredEvent - if event.startswith('libvirt.'): - raise exceptions.IgnoredEvent - if event.startswith('metrics.'): - raise exceptions.IgnoredEvent - if event.startswith('scheduler.'): - raise exceptions.IgnoredEvent - if event.startswith('server_group.'): - raise exceptions.IgnoredEvent - if event.startswith('service.'): - raise exceptions.IgnoredEvent - if event == 'volume.usage': - raise exceptions.IgnoredEvent - - raise exceptions.UnsupportedEventType - - diff --git a/test-requirements.txt b/test-requirements.txt index ace7b0a..a46e5dc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,8 @@ before_after flake8 freezegun pylint +pylint-flask +pylint-flask-sqlalchemy pytest pytest-cov pytest-flask diff --git a/tox.ini b/tox.ini index 9054eac..bf4c8a9 100644 --- a/tox.ini +++ b/tox.ini @@ -21,8 +21,11 @@ commands = {posargs} [testenv:linters] commands = - pylint atmosphere - flake8 atmosphere + pylint atmosphere \ + --load-plugins pylint_flask,pylint_flask_sqlalchemy \ + --ignore migrations,tests + flake8 atmosphere \ + --exclude .tox,atmosphere/migrations,atmosphere/tests [testenv:docs] deps =