From 75ab643e197f673f23f2afe75470ee8bef376003 Mon Sep 17 00:00:00 2001
From: Mohammed Naser <mnaser@vexxhost.com>
Date: Thu, 25 Mar 2021 22:05:35 -0400
Subject: [PATCH] Add support for Cinder volumes
Change-Id: I33401c86051293d5246856451571e4827be833ed
---
atmosphere/models.py | 57 +++++++++-
atmosphere/tests/unit/api/test_ingress.py | 14 +--
atmosphere/tests/unit/conftest.py | 3 +-
atmosphere/tests/unit/fake.py | 40 ++++++-
atmosphere/tests/unit/test_models.py | 126 ++++++++++++++++------
atmosphere/tests/unit/test_utils.py | 8 +-
contrib/event_definitions.yaml | 3 -
7 files changed, 200 insertions(+), 51 deletions(-)
diff --git a/atmosphere/models.py b/atmosphere/models.py
index c92beb6..3d03fa5 100644
--- a/atmosphere/models.py
+++ b/atmosphere/models.py
@@ -46,6 +46,8 @@ def get_model_type_from_event(event):
"""get_model_type_from_event"""
if event.startswith('compute.instance'):
return Instance, InstanceSpec
+ if event.startswith('volume.'):
+ return Volume, VolumeSpec
if event.startswith('aggregate.'):
raise exceptions.IgnoredEvent
if event.startswith('compute_task.'):
@@ -66,8 +68,6 @@ def get_model_type_from_event(event):
raise exceptions.IgnoredEvent
if event.startswith('service.'):
raise exceptions.IgnoredEvent
- if event == 'volume.usage':
- raise exceptions.IgnoredEvent
raise exceptions.UnsupportedEventType
@@ -198,7 +198,11 @@ class Resource(db.Model, GetOrCreateMixin):
# If we're deleted, then we close the current period.
if resource.__class__.is_event_delete(event):
- period.ended_at = event['traits']['deleted_at']
+ # NOTE(mnaser): Some resources don't have `deleted_at`, so we
+ # resort to the timestamp of the event instead.
+ period.ended_at = event['traits'].get(
+ 'deleted_at', event['generated']
+ )
elif period.spec != spec:
period.ended_at = event['generated']
@@ -265,6 +269,26 @@ class Instance(Resource):
return 'deleted_at' in event['traits']
+class Volume(Resource):
+ """Volume"""
+
+ STATE_ALLOW_LIST = ('available', 'deleted')
+
+ __mapper_args__ = {
+ 'polymorphic_identity': 'OS::Cinder::Volume'
+ }
+
+ @classmethod
+ def is_event_ignored(cls, event):
+ """is_event_ignored"""
+ return event['traits']['state'] not in cls.STATE_ALLOW_LIST
+
+ @classmethod
+ def is_event_delete(cls, event):
+ """is_event_delete"""
+ return event['traits']['state'] == 'deleted'
+
+
class BigIntegerDateTime(TypeDecorator):
"""BigIntegerDateTime"""
@@ -368,3 +392,30 @@ class InstanceSpec(Spec):
'instance_type': self.instance_type,
'state': self.state,
}
+
+
+class VolumeSpec(Spec):
+ """VolumeSpec"""
+
+ id = db.Column(db.Integer, db.ForeignKey('spec.id'), primary_key=True)
+ volume_type = db.Column(db.String(255))
+ volume_size = db.Column(db.String(255))
+ state = db.Column(db.String(255))
+
+ __table_args__ = (
+ db.UniqueConstraint('volume_type', 'volume_size', 'state'),
+ )
+
+ __mapper_args__ = {
+ 'polymorphic_identity': 'OS::Cinder::Volume',
+ }
+
+ @property
+ def serialize(self):
+ """Return object data in easily serializable format"""
+
+ return {
+ 'volume_type': self.volume_type,
+ 'volume_size': self.volume_size,
+ 'state': self.state,
+ }
diff --git a/atmosphere/tests/unit/api/test_ingress.py b/atmosphere/tests/unit/api/test_ingress.py
index 1cf187f..489d0fb 100644
--- a/atmosphere/tests/unit/api/test_ingress.py
+++ b/atmosphere/tests/unit/api/test_ingress.py
@@ -44,7 +44,7 @@ class TestEvent:
assert response.status_code == 400
def test_with_one_event_provided(self, client):
- event = fake.get_event()
+ event = fake.get_instance_event()
response = client.post('/v1/event', json=[event])
assert response.status_code == 204
@@ -53,8 +53,8 @@ class TestEvent:
assert models.Spec.query.count() == 1
def test_with_multiple_events_provided(self, client):
- event_1 = fake.get_event(resource_id='fake-resource-1')
- event_2 = fake.get_event(resource_id='fake-resource-2')
+ event_1 = fake.get_instance_event(resource_id='fake-resource-1')
+ event_2 = fake.get_instance_event(resource_id='fake-resource-2')
response = client.post('/v1/event', json=[event_1, event_2])
@@ -64,7 +64,7 @@ class TestEvent:
assert models.Spec.query.count() == 1
def test_with_old_event_provided(self, client):
- event_new = fake.get_event()
+ event_new = fake.get_instance_event()
event_new['generated'] = '2020-06-07T01:42:54.736337'
response = client.post('/v1/event', json=[event_new])
@@ -73,7 +73,7 @@ class TestEvent:
assert models.Period.query.count() == 1
assert models.Spec.query.count() == 1
- event_old = fake.get_event()
+ event_old = fake.get_instance_event()
event_old['generated'] = '2020-06-07T01:40:54.736337'
response = client.post('/v1/event', json=[event_old])
@@ -83,7 +83,7 @@ class TestEvent:
assert models.Spec.query.count() == 1
def test_with_invalid_event_provided(self, client):
- event = fake.get_event(event_type='foo.bar.exists')
+ event = fake.get_instance_event(event_type='foo.bar.exists')
response = client.post('/v1/event', json=[event])
assert response.status_code == 400
@@ -92,7 +92,7 @@ class TestEvent:
assert models.Spec.query.count() == 0
def test_with_ignored_event_provided(self, client, ignored_event):
- event = fake.get_event(event_type=ignored_event)
+ event = fake.get_instance_event(event_type=ignored_event)
response = client.post('/v1/event', json=[event])
assert response.status_code == 202
diff --git a/atmosphere/tests/unit/conftest.py b/atmosphere/tests/unit/conftest.py
index cb009f3..74a97d2 100644
--- a/atmosphere/tests/unit/conftest.py
+++ b/atmosphere/tests/unit/conftest.py
@@ -30,8 +30,7 @@ from atmosphere.api import ingress
'metrics.update',
'scheduler.select_destinations.end',
'server_group.add_member',
- 'service.create',
- 'volume.usage',
+ 'service.create'
])
def ignored_event(request):
yield request.param
diff --git a/atmosphere/tests/unit/fake.py b/atmosphere/tests/unit/fake.py
index 39126f3..52706f0 100644
--- a/atmosphere/tests/unit/fake.py
+++ b/atmosphere/tests/unit/fake.py
@@ -20,7 +20,7 @@ from atmosphere import models
from atmosphere import utils
-def get_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
+def get_instance_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
return dict({
'generated': '2020-06-07T01:42:54.736337',
'event_type': event_type,
@@ -36,8 +36,32 @@ def get_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
})
-def get_normalized_event():
- event = get_event()
+def get_volume_event(resource_id='fake-uuid', event_type='volume.exists'):
+ return dict({
+ 'generated': '2020-06-07T01:42:54.736337',
+ 'event_type': event_type,
+ 'traits': [
+ ["service", 1, "volume.ironic-devstack"],
+ ["request_id", 1, "req-66a05c73-964e-4d12-a0c3-7d3b0d5801ce"],
+ ["project_id", 1, "a1da863b589642558b7a87f09840a565"],
+ ["user_id", 1, "a6db2097a75a4cf3b3b4336108017aae"],
+ ["tenant_id", 1, "a1da863b589642558b7a87f09840a565"],
+ ["resource_id", 1, "3c1e0499-7621-496c-bab4-59a7053f8b59"],
+ ["volume_type", 1, "7d233c12-d346-4948-8901-7afd5c5dd590"],
+ ["volume_size", 2, 1],
+ ["state", 1, "available"],
+ ["created_at", 4, "2021-03-26T00:36:28"],
+ ]
+ })
+
+
+def get_normalized_instance_event():
+ event = get_instance_event()
+ return utils.normalize_event(event)
+
+
+def get_normalized_volume_event():
+ event = get_volume_event()
return utils.normalize_event(event)
@@ -53,6 +77,16 @@ def get_instance_spec(**kwargs):
return models.InstanceSpec(**kwargs)
+def get_volume_spec(**kwargs):
+ if not kwargs:
+ kwargs = {
+ 'volume_type': '7d233c12-d346-4948-8901-7afd5c5dd590',
+ 'volume_size': 3,
+ 'state': 'available'
+ }
+ return models.VolumeSpec(**kwargs)
+
+
def get_resource_with_periods(number):
resource = get_resource()
diff --git a/atmosphere/tests/unit/test_models.py b/atmosphere/tests/unit/test_models.py
index 1caae82..7fb10bf 100644
--- a/atmosphere/tests/unit/test_models.py
+++ b/atmosphere/tests/unit/test_models.py
@@ -46,7 +46,7 @@ def _db(app):
class GetOrCreateTestMixin:
def test_with_existing_object(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
assert self.MODEL.query_from_event(event).count() == 0
old_object = self.MODEL.get_or_create(event)
@@ -58,14 +58,14 @@ class GetOrCreateTestMixin:
assert old_object == new_object
def test_with_no_existing_object(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
assert self.MODEL.query_from_event(event).count() == 0
new_object = self.MODEL.get_or_create(event)
assert self.MODEL.query_from_event(event).count() == 1
def test_with_object_created_during_creation(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
assert self.MODEL.query_from_event(event).count() == 0
def before_session_begin(*args, **kwargs):
@@ -89,7 +89,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_by_project(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
start = event['traits']['created_at'] - relativedelta(hours=+1)
@@ -105,7 +105,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 3600
def test_get_all_by_time_range_with_resource_ended_before_start(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@@ -118,7 +118,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_with_resource_started_after_end(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
ended = event['traits']['created_at'] - relativedelta(hours=+1)
@@ -128,7 +128,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_with_active_resource_after_start(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
start = event['traits']['created_at'] - relativedelta(hours=+1)
@@ -139,7 +139,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 3600
def test_get_all_by_time_range_with_active_resource_before_start(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
start = event['traits']['created_at'] + relativedelta(minutes=+30)
@@ -150,7 +150,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 1800
def test_get_all_by_time_range_with_active_resource_after_end(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@@ -163,7 +163,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_with_resource_inside_range(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(minutes=+15)
@@ -177,7 +177,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 900
def test_get_all_by_time_range_with_resource_with_multiple_periods(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['created_at'] = event['traits']['created_at'] + \
relativedelta(microseconds=0)
models.Resource.get_or_create(event)
@@ -196,7 +196,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[1].seconds == 2700
def test_get_all_by_time_range_with_resource_with_one_active_period(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['created_at'] = event['traits']['created_at'] + \
relativedelta(microseconds=0)
models.Resource.get_or_create(event)
@@ -215,7 +215,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 2700
def test_from_event(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
resource = models.Resource.from_event(event)
assert resource.uuid == event['traits']['resource_id']
@@ -226,7 +226,7 @@ class TestResource(GetOrCreateTestMixin):
def test_query_from_event(self, mock_query_property_getter):
mock_filter_by = mock_query_property_getter.return_value.filter_by
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
query = models.Resource.query_from_event(event)
mock_filter_by.assert_called_with(
@@ -235,7 +235,7 @@ class TestResource(GetOrCreateTestMixin):
)
def test_get_or_create_with_old_event(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
new_object = models.Resource.get_or_create(event)
old_event = event.copy()
@@ -246,7 +246,7 @@ class TestResource(GetOrCreateTestMixin):
models.Resource.get_or_create(old_event)
def test_get_or_create_refresh_updated_at(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
old_object = models.Resource.get_or_create(event)
new_event = event.copy()
@@ -259,14 +259,14 @@ class TestResource(GetOrCreateTestMixin):
assert models.Resource.query_from_event(event).count() == 1
def test_get_or_create_using_created_at(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
assert resource.get_open_period().started_at == \
event['traits']['created_at']
def test_get_or_create_using_deleted_event_only(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@@ -278,7 +278,7 @@ class TestResource(GetOrCreateTestMixin):
assert resource.periods[0].seconds == 3600
def test_get_or_create_using_multiple_deleted_events(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@@ -287,7 +287,7 @@ class TestResource(GetOrCreateTestMixin):
models.Resource.get_or_create(event)
def test_get_or_create_using_deleted_event(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
old_resource = models.Resource.get_or_create(event)
assert old_resource.get_open_period() is not None
@@ -305,7 +305,7 @@ class TestResource(GetOrCreateTestMixin):
assert new_resource.periods[0].seconds == 3600
def test_get_or_create_using_updated_spec(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
old_resource = models.Resource.get_or_create(event)
assert old_resource.get_open_period() is not None
@@ -323,7 +323,7 @@ class TestResource(GetOrCreateTestMixin):
assert new_resource.get_open_period().started_at == event['generated']
def test_get_or_create_using_same_spec(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
old_resource = models.Resource.get_or_create(event)
assert old_resource.get_open_period() is not None
@@ -429,33 +429,33 @@ class TestResource(GetOrCreateTestMixin):
@pytest.mark.usefixtures("db_session")
class TestInstance:
def test_is_event_delete(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
assert models.Instance.is_event_delete(event) == False
def test_is_event_delete_for_actual_delete(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['generated']
assert models.Instance.is_event_delete(event) == True
def test_is_event_ignored(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
assert models.Instance.is_event_ignored(event) == False
def test_is_event_ignored_for_pending_delete(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['event_type'] = 'compute.instance.delete.start'
event['traits']['state'] = 'deleted'
assert models.Instance.is_event_ignored(event) == True
def test_is_event_ignored_for_deleted(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
event['event_type'] = 'compute.instance.delete.start'
event['traits']['state'] = 'deleted'
event['traits']['deleted_at'] = event['generated']
assert models.Instance.is_event_ignored(event) == False
- def test_get_or_create_has_no_deleted_period(self):
- event = fake.get_normalized_event()
+ def _test_get_or_create_has_no_deleted_period(self, event, delete_event):
+ event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
assert resource.get_open_period() is not None
@@ -479,6 +479,59 @@ class TestInstance:
assert len(resource.periods) == 1
+@pytest.mark.usefixtures("db_session")
+class TestVolume:
+ def test_is_event_delete(self):
+ event = fake.get_normalized_volume_event()
+ assert models.Volume.is_event_delete(event) == False
+
+ def test_is_event_delete_for_actual_delete(self):
+ event = fake.get_normalized_volume_event()
+ event['traits']['state'] = 'deleted'
+ assert models.Volume.is_event_delete(event) == True
+
+ def test_is_event_ignored(self):
+ event = fake.get_normalized_volume_event()
+ assert models.Volume.is_event_ignored(event) == False
+
+ def test_is_event_ignored_for_pending_delete(self):
+ event = fake.get_normalized_instance_event()
+ event['event_type'] = 'volume.delete.start'
+ event['traits']['state'] = 'deleting'
+ assert models.Volume.is_event_ignored(event) == True
+
+ def test_is_event_ignored_for_pending_create(self):
+ event = fake.get_normalized_instance_event()
+ event['event_type'] = 'volume.delete.start'
+ event['traits']['state'] = 'creating'
+ assert models.Volume.is_event_ignored(event) == True
+
+ def _test_get_or_create_has_no_deleted_period(self, event, delete_event):
+ event = fake.get_normalized_instance_event()
+ resource = models.Resource.get_or_create(event)
+
+ assert resource.get_open_period() is not None
+ assert len(resource.periods) == 1
+
+ event['event_type'] = 'volume.delete.start'
+ event['traits']['state'] = 'deleting'
+ event['generated'] += relativedelta(hours=+1)
+
+ with pytest.raises(exceptions.IgnoredEvent) as e:
+ models.Resource.get_or_create(event)
+
+ assert resource.get_open_period() is not None
+ assert len(resource.periods) == 1
+
+ event['event_type'] = 'volume.delete.end'
+ event['traits']['state'] = 'deleted'
+ event['generated'] += relativedelta(seconds=+2)
+ resource = models.Resource.get_or_create(event)
+
+ assert resource.get_open_period() is None
+ assert len(resource.periods) == 1
+
+
@pytest.mark.usefixtures("db_session")
class TestPeriod:
def test_serialize_without_start(self):
@@ -541,7 +594,7 @@ class TestSpec(GetOrCreateTestMixin):
MODEL = models.Spec
def test_from_event(self):
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
spec = models.Spec.from_event(event)
assert spec.instance_type == 'v1-standard-1'
@@ -551,7 +604,7 @@ class TestSpec(GetOrCreateTestMixin):
def test_query_from_event(self, mock_query_property_getter):
mock_filter_by = mock_query_property_getter.return_value.filter_by
- event = fake.get_normalized_event()
+ event = fake.get_normalized_instance_event()
query = models.Spec.query_from_event(event)
mock_filter_by.assert_called_with(
@@ -569,3 +622,14 @@ class TestInstanceSpec:
'instance_type': spec.instance_type,
'state': spec.state,
}
+
+@pytest.mark.usefixtures("db_session")
+class TestVolumeSpec:
+ def test_serialize(self):
+ spec = fake.get_volume_spec()
+
+ assert spec.serialize == {
+ 'volume_type': spec.volume_type,
+ 'volume_size': spec.volume_size,
+ 'state': spec.state,
+ }
diff --git a/atmosphere/tests/unit/test_utils.py b/atmosphere/tests/unit/test_utils.py
index 99b180d..71189a2 100644
--- a/atmosphere/tests/unit/test_utils.py
+++ b/atmosphere/tests/unit/test_utils.py
@@ -24,8 +24,8 @@ from atmosphere import utils
class TestNormalizeEvent:
def test_normalize_event(self):
- event = fake.get_event()
- event_expected = fake.get_event()
+ event = fake.get_instance_event()
+ event_expected = fake.get_instance_event()
event_expected.update({
"generated": datetime.datetime(2020, 6, 7, 1, 42, 54, 736337),
"traits": {
@@ -47,6 +47,10 @@ class TestModelTypeDetection:
assert models.get_model_type_from_event('compute.instance.exists') == \
(models.Instance, models.InstanceSpec)
+ def test_volume(self):
+ assert models.get_model_type_from_event('volume.create.start') == \
+ (models.Volume, models.VolumeSpec)
+
def test_ignored_resource(self, ignored_event):
with pytest.raises(exceptions.IgnoredEvent) as e:
models.get_model_type_from_event(ignored_event)
diff --git a/contrib/event_definitions.yaml b/contrib/event_definitions.yaml
index 3204546..6c47258 100644
--- a/contrib/event_definitions.yaml
+++ b/contrib/event_definitions.yaml
@@ -31,7 +31,4 @@
created_at:
type: datetime
fields: payload.created_at
- deleted_at:
- type: datetime
- fields: payload.deleted_at
...
\ No newline at end of file