From 3d09835ea729d2b7c6089de21dcd76916e3599ab Mon Sep 17 00:00:00 2001
From: flavien peyre <peyre.flavien@gmail.com>
Date: Wed, 15 Jul 2015 13:35:30 -0400
Subject: [PATCH] Add event paging
Change-Id: Ifaa82db925ca954fea19a91de43f07ff15f8ebd8
---
surveil/api/handlers/status/event_handler.py | 43 +----------
surveil/api/handlers/status/influxdb_query.py | 75 ++++++++++++++++++-
.../api/controllers/v2/status/test_events.py | 70 +++++++++++++----
.../v2/status/test_hosts_metric.py | 2 +-
.../v2/status/test_hosts_services_metrics.py | 2 +-
5 files changed, 132 insertions(+), 60 deletions(-)
diff --git a/surveil/api/handlers/status/event_handler.py b/surveil/api/handlers/status/event_handler.py
index 7feb8ee..1704037 100644
--- a/surveil/api/handlers/status/event_handler.py
+++ b/surveil/api/handlers/status/event_handler.py
@@ -23,43 +23,6 @@ class EventHandler(handler.Handler):
def get_all(self, live_query=None):
"""Return all logs."""
influx_client = self.request.influxdb_client
- query = influxdb_query.build_influxdb_query(live_query, "EVENT")
- response = influx_client.query(query)
-
- events = []
-
- for item in response.items():
- tags = item[0][1]
- for point in response.get_points(tags=tags):
- point.update(tags)
- event_dict = self._event_dict_from_influx_item(point)
- events.append(event.Event(**event_dict))
-
- return events
-
- def _event_dict_from_influx_item(self, item):
- mappings = [
- 'time',
- 'event_type',
- 'host_name',
- 'service_description',
- 'state',
- 'state_type',
- 'attempts',
- 'downtime_type',
- 'notification_type',
- 'notification_method',
- 'contact',
- 'alert_type',
- 'output',
- 'acknowledgement'
- ]
-
- event_dict = {}
-
- for field in mappings:
- value = item.get(field, None)
- if value is not None and value != "":
- event_dict[field] = value
-
- return event_dict
+ query = influxdb_query.build_influxdb_query(live_query, "EVENT",
+ multiple_series=True)
+ return influxdb_query.paging(influx_client.query(query), event.Event, live_query)
\ No newline at end of file
diff --git a/surveil/api/handlers/status/influxdb_query.py b/surveil/api/handlers/status/influxdb_query.py
index d212d3c..b116764 100644
--- a/surveil/api/handlers/status/influxdb_query.py
+++ b/surveil/api/handlers/status/influxdb_query.py
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import functools
import json
@@ -20,8 +21,8 @@ def build_influxdb_query(live_query,
group_by=[],
order_by=[],
additional_filters={},
- limit=None):
-
+ limit=None,
+ multiple_series=False):
query = ['SELECT * FROM', measurement]
filters = {}
@@ -33,8 +34,11 @@ def build_influxdb_query(live_query,
if live_query.time_interval:
time = live_query.time_interval
if live_query.paging:
- limit = live_query.paging.size
- offset = limit * live_query.paging.page
+ if multiple_series:
+ limit = live_query.paging.size * (live_query.paging.page + 1)
+ else:
+ limit = live_query.paging.size
+ offset = (live_query.paging.page + 1) * live_query.paging.size
filters.update(additional_filters)
query += _build_where_clause(filters, time)
@@ -90,3 +94,66 @@ def _build_where_clause(filters, time=None):
value))
return clause
+
+
+def paging(response, datamodel, live_query=None):
+ """Paging function
+
+ :param response: a python-influxdb resulset
+ :param datamodel: an Surveil API datamodel class
+ :param live_query: an influxdb_query
+ :return: a dict of datamodel object. If the live query contain paging,
+ the dict is sorted by datamodel time attribute and contain
+ live_query.paging.size object for the live_query.paging.page page
+ """
+ if live_query and live_query.paging:
+ limit_paging = live_query.paging.size * (live_query.paging.page + 1)
+ limit = live_query.paging.size + live_query.paging.page
+ offset_paging = live_query.paging.page * live_query.paging.size
+
+ def sort_by_time(init, point_tag):
+ event = {}
+ event.update(point_tag[0])
+ event.update(point_tag[1])
+ init.append(datamodel(**event))
+ init.sort(key=lambda event: event.time,
+ reverse=True)
+ return init[:limit_paging]
+
+ response = [(tag[1], _dict_from_influx_item(datamodel, point))
+ for tag, points in response.items()
+ for point in points]
+
+ event_list = functools.reduce(sort_by_time, response, [])
+
+ return event_list[offset_paging:limit+1]
+
+ else:
+ events = []
+
+ for item in response.items():
+ tags = item[0][1]
+ for point in response.get_points(tags=tags):
+ point.update(tags)
+ event_dict = _dict_from_influx_item(datamodel, point)
+ events.append(datamodel(**event_dict))
+
+ return events
+
+
+def _dict_from_influx_item(datamodel, item):
+ """Create a dict representing a python-influxdb item
+
+ :param item: an python influxdb item object
+ :param datamodel: an Surveil API datamodel class
+ :return: a dict (datamodel_attribute:item_value)
+
+ >>> _event_dict_from_influx_item(Event, {"time": 4})
+ {'time': 4}
+ >>> _event_dict_from_influx_item(Event, {"time": "null"})
+ {}
+ """
+
+ fields = [attr.name for attr in getattr(datamodel, "_wsme_attributes")]
+ return dict([(field, item.get(field, None)) for field in fields
+ if item.get(field, None) is not None])
diff --git a/surveil/tests/api/controllers/v2/status/test_events.py b/surveil/tests/api/controllers/v2/status/test_events.py
index 09996c8..e63728c 100644
--- a/surveil/tests/api/controllers/v2/status/test_events.py
+++ b/surveil/tests/api/controllers/v2/status/test_events.py
@@ -44,7 +44,7 @@ class TestEvents(functionalTest.FunctionalTest):
],
"values": [
[
- "2015-06-04T18:55:12Z",
+ "2015-06-04T18:55:19Z",
1,
"Connection refused",
"CRITICAL",
@@ -52,7 +52,7 @@ class TestEvents(functionalTest.FunctionalTest):
"SERVICE"
],
[
- '2015-06-04T18:55:12Z',
+ '2015-06-04T18:55:18Z',
2,
'Connection refused',
'CRITICAL',
@@ -60,7 +60,7 @@ class TestEvents(functionalTest.FunctionalTest):
'SERVICE'
],
[
- '2015-06-04T18:55:12Z',
+ '2015-06-04T18:55:17Z',
3,
'Connection refused',
'CRITICAL',
@@ -86,7 +86,7 @@ class TestEvents(functionalTest.FunctionalTest):
],
'values': [
[
- '2015-06-04T18:55:12Z',
+ '2015-06-04T18:55:16Z',
1,
'Warning - Connection refused',
'CRITICAL',
@@ -94,7 +94,7 @@ class TestEvents(functionalTest.FunctionalTest):
'SERVICE'
],
[
- '2015-06-04T18:55:12Z',
+ '2015-06-04T18:55:15Z',
2,
'Warning - Connection refused',
'WARNING',
@@ -120,7 +120,7 @@ class TestEvents(functionalTest.FunctionalTest):
],
'values': [
[
- '2015-06-04T18:55:12Z',
+ '2015-06-04T18:55:14Z',
'SERVICE',
'admin',
'CRITICAL',
@@ -128,7 +128,7 @@ class TestEvents(functionalTest.FunctionalTest):
None
],
[
- '2015-06-04T18:55:12Z',
+ '2015-06-04T18:55:13Z',
'SERVICE',
'admin',
'CRITICAL',
@@ -174,7 +174,7 @@ class TestEvents(functionalTest.FunctionalTest):
"host_name": "myServiceIsDown",
"event_type": "ALERT",
"service_description": "iAmADownService",
- "time": "2015-06-04T18:55:12Z",
+ "time": "2015-06-04T18:55:19Z",
"attempts": 1,
"output": "Connection refused",
"state": "CRITICAL",
@@ -185,7 +185,7 @@ class TestEvents(functionalTest.FunctionalTest):
'host_name': 'myServiceIsDown',
'event_type': 'ALERT',
'service_description': 'iAmADownService',
- 'time': '2015-06-04T18:55:12Z',
+ 'time': '2015-06-04T18:55:18Z',
'attempts': 2,
'output': 'Connection refused',
'state': 'CRITICAL',
@@ -196,7 +196,7 @@ class TestEvents(functionalTest.FunctionalTest):
'host_name': 'myServiceIsDown',
'event_type': 'ALERT',
'service_description': 'iAmADownService',
- 'time': '2015-06-04T18:55:12Z',
+ 'time': '2015-06-04T18:55:17Z',
'attempts': 3,
'output': 'Connection refused',
'state': 'CRITICAL',
@@ -207,7 +207,7 @@ class TestEvents(functionalTest.FunctionalTest):
'host_name': 'savoirfairelinux',
'event_type': 'ALERT',
'service_description': 'CPU',
- 'time': '2015-06-04T18:55:12Z',
+ 'time': '2015-06-04T18:55:16Z',
'attempts': 1,
'output': 'Warning - Connection refused',
'state': 'CRITICAL',
@@ -218,7 +218,7 @@ class TestEvents(functionalTest.FunctionalTest):
'host_name': 'savoirfairelinux',
'event_type': 'ALERT',
'service_description': 'CPU',
- 'time': '2015-06-04T18:55:12Z',
+ 'time': '2015-06-04T18:55:15Z',
'attempts': 2,
'output': 'Warning - Connection refused',
'state': 'WARNING',
@@ -229,7 +229,7 @@ class TestEvents(functionalTest.FunctionalTest):
'host_name': 'savoirfairelinux',
'event_type': 'NOTIFICATION',
'service_description': 'CPU',
- 'time': '2015-06-04T18:55:12Z',
+ 'time': '2015-06-04T18:55:14Z',
'notification_type': 'SERVICE',
'contact': 'admin',
'state': 'CRITICAL',
@@ -239,7 +239,7 @@ class TestEvents(functionalTest.FunctionalTest):
'host_name': 'savoirfairelinux',
'event_type': 'NOTIFICATION',
'service_description': 'CPU',
- 'time': '2015-06-04T18:55:12Z',
+ 'time': '2015-06-04T18:55:13Z',
'notification_type': 'SERVICE',
'contact': 'admin',
'state': 'CRITICAL',
@@ -344,3 +344,45 @@ class TestEvents(functionalTest.FunctionalTest):
'notification_method': 'notify-service-by-email'
}]
)
+
+ def test_paging(self):
+ with requests_mock.Mocker() as m:
+ m.register_uri(requests_mock.GET,
+ 'http://influxdb:8086/query',
+ text=self.influxdb_response)
+
+ query = {'paging': {'page': 1, 'size': 2}}
+
+ response = self.post_json('/v2/status/events', params=query)
+
+ self.assertEqual(
+ m.last_request.qs['q'],
+ ["select * from event limit 4"]
+ )
+
+ self.assert_count_equal_backport(
+ json.loads(response.body.decode()), [
+
+ {
+ 'host_name': 'myServiceIsDown',
+ 'event_type': 'ALERT',
+ 'service_description': 'iAmADownService',
+ 'time': '2015-06-04T18:55:17Z',
+ 'attempts': 3,
+ 'output': 'Connection refused',
+ 'state': 'CRITICAL',
+ 'state_type': 'SOFT',
+ 'alert_type': 'SERVICE'
+ },
+ {
+ 'host_name': 'savoirfairelinux',
+ 'event_type': 'ALERT',
+ 'service_description': 'CPU',
+ 'time': '2015-06-04T18:55:16Z',
+ 'attempts': 1,
+ 'output': 'Warning - Connection refused',
+ 'state': 'CRITICAL',
+ 'state_type': 'HARD',
+ 'alert_type': 'SERVICE'
+ }]
+ )
\ No newline at end of file
diff --git a/surveil/tests/api/controllers/v2/status/test_hosts_metric.py b/surveil/tests/api/controllers/v2/status/test_hosts_metric.py
index 9747a5c..f136712 100644
--- a/surveil/tests/api/controllers/v2/status/test_hosts_metric.py
+++ b/surveil/tests/api/controllers/v2/status/test_hosts_metric.py
@@ -4,7 +4,7 @@
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
diff --git a/surveil/tests/api/controllers/v2/status/test_hosts_services_metrics.py b/surveil/tests/api/controllers/v2/status/test_hosts_services_metrics.py
index f55a516..f6bb2fd 100644
--- a/surveil/tests/api/controllers/v2/status/test_hosts_services_metrics.py
+++ b/surveil/tests/api/controllers/v2/status/test_hosts_services_metrics.py
@@ -230,7 +230,7 @@ class TestHostMetric(functionalTest.FunctionalTest):
"and host_name='srv-monitoring-01' "
"and service_description='load' "
"order by time desc "
- "limit 10 offset 30"
+ "limit 10 offset 40"
]
)
self.assert_count_equal_backport(