x/surveil
Code Issues Proposed changes

Service status API now uses live_shinken

Browse Source 
Change-Id: I8a0b986e3ee3bc926b8daa9ea818bc7a2f7f679e
This commit is contained in:
aviau 2015-05-11 11:01:40 -04:00
parent 39f1a88a9a
commit cfc49075d3
12 changed files with 350 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
surveil/api/datamodel/status/live_query.py
View File

@ -26,13 +26,13 @@ class LiveQuery(types.Base):
filters = wsme.wsattr(wtypes.text, mandatory=True)
"The filter expression encoded in json."
fields = wsme.wsattr(wtypes.text, mandatory=False)
fields = wsme.wsattr([wtypes.text], mandatory=False)
"List of fields to include in the response."
@classmethod
def sample(cls):
return cls(
fields=json.dumps(['host_name', 'last_check']),
fields=['host_name', 'last_check'],
filters=json.dumps({
"isnot": {
"state": ["0", "1"],

11
surveil/api/datamodel/status/live_service.py
View File

@ -28,10 +28,10 @@ class LiveService(types.Base):
description = wsme.wsattr(wtypes.text, mandatory=False)
"""The description of the sevice"""
state = wsme.wsattr(int, mandatory=False)
state = wsme.wsattr(wtypes.text, mandatory=False)
"""The current state of the service"""
acknowledged = wsme.wsattr(int, mandatory=False)
acknowledged = wsme.wsattr(bool, mandatory=False)
"""Wether or not the problem, if any, has been acknowledged"""
last_check = wsme.wsattr(int, mandatory=False)
@ -47,10 +47,11 @@ class LiveService(types.Base):
def sample(cls):
return cls(
host_name='Webserver',
service_name='Apache',
service_description='Apache',
description='Serves Stuff',
state=0,
state='OK',
last_check=1429220785,
last_state_change=1429220785,
plugin_output='HTTP OK - GOT NICE RESPONSE'
plugin_output='HTTP OK - GOT NICE RESPONSE',
acknowledged=True,
)

32
surveil/api/handlers/status/fields_filter.py Normal file
View File

@ -0,0 +1,32 @@
# Copyright 2014 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import wsme
def filter_fields(item_list, live_query):
"""Takes unwanted keys out of a dict depending on a live_query."""
filtered_items = []
if live_query.fields != wsme.Unset:
fields = live_query.fields
for item in item_list:
filtered_item = {}
for field in fields:
filtered_item[field] = item.get(field, None)
filtered_items.append(filtered_item)
else:
filtered_items = item_list
return filtered_items

18
surveil/api/handlers/status/influxdb_query.py
View File

@ -14,24 +14,6 @@
import json
import wsme
def filter_fields(item_list, live_query):
filtered_items = []
if live_query.fields != wsme.Unset:
fields = json.loads(live_query.fields)
for item in item_list:
filtered_item = {}
for field in fields:
filtered_item[field] = item[field]
filtered_items.append(filtered_item)
else:
filtered_items = item_list
return filtered_items
def build_influxdb_query(live_query,
measurement,

3
surveil/api/handlers/status/live_host_handler.py
View File

@ -17,6 +17,7 @@ import json
from surveil.api.datamodel.status import live_host
from surveil.api.handlers import handler
from surveil.api.handlers.status import fields_filter
from surveil.api.handlers.status import influxdb_query
@ -57,7 +58,7 @@ class HostHandler(handler.Handler):
host_dicts.append(host_dict)
if live_query:
host_dicts = influxdb_query.filter_fields(
host_dicts = fields_filter.filter_fields(
host_dicts,
live_query
)

126
surveil/api/handlers/status/live_service_handler.py
View File

@ -12,56 +12,48 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import json
from surveil.api.datamodel.status import live_service
from surveil.api.handlers import handler
from surveil.api.handlers.status import influxdb_query
from surveil.api.handlers.status import mongodb_query
import wsme
class ServiceHandler(handler.Handler):
"""Fulfills a request on live services."""
def get(self, host_name, service_name):
def get(self, host_name, service_description):
"""Return a specific service."""
cli = self.request.influxdb_client
query = ("SELECT * from LIVE_SERVICE_STATE "
"WHERE host_name='%s' "
"AND service_description='%s' "
"GROUP BY * "
"ORDER BY time DESC "
"LIMIT 1") % (host_name, service_name)
response = cli.query(query)
host = live_service.LiveService(
**self._service_dict_from_influx_item(response.items()[0])
mongo_s = self.request.mongo_connection.shinken_live.services.find_one(
{"host_name": host_name,
"service_description": service_description},
)
return host
return live_service.LiveService(**mongo_s)
def get_all(self, live_query=None):
"""Return all live services."""
cli = self.request.influxdb_client
query = influxdb_query.build_influxdb_query(
live_query,
'LIVE_SERVICE_STATE',
group_by=['host_name', 'service_description'],
order_by=['time DESC'],
limit=1
)
response = cli.query(query)
service_dicts = []
for item in response.items():
service_dict = self._service_dict_from_influx_item(item)
service_dicts.append(service_dict)
if live_query:
service_dicts = influxdb_query.filter_fields(
service_dicts,
live_query
)
lq_filters, lq_fields = _translate_live_query(live_query)
else:
lq_filters = {}
lq_fields = {}
query, fields = mongodb_query.build_mongodb_query(lq_filters,
lq_fields)
if fields != {}:
mongo_dicts = (self.request.mongo_connection.
shinken_live.services.find(query, fields))
else:
mongo_dicts = (self.request.mongo_connection.
shinken_live.services.find(query))
service_dicts = [
_service_dict_from_mongo_item(s) for s in mongo_dicts
]
services = []
for service_dict in service_dicts:
@ -70,20 +62,54 @@ class ServiceHandler(handler.Handler):
return services
def _service_dict_from_influx_item(self, item):
tags = item[0][1]
points = item[1]
first_point = next(points)
service_dict = {
"service_description": tags['service_description'],
"host_name": tags['host_name'],
"description": tags['service_description'],
"state": first_point['state'],
"acknowledged": int(first_point['acknowledged']),
"last_check": int(first_point['last_check']),
"last_state_change": int(first_point['last_state_change']),
"plugin_output": first_point['output']
}
def _translate_live_query(live_query):
"""Translate field names in a live query so that they match mongodb."""
return service_dict
# Mappings
mapping = {
"last_check": "last_chk",
"description": "service_description",
"plugin_output": "output",
"acknowledged": "problem_has_been_acknowledged",
}
# Load the fields
if live_query.fields != wsme.Unset:
fields = live_query.fields
else:
fields = []
# Translate the fields
lq_fields = []
for field in fields:
lq_fields.append(mapping.get(field, field))
# Load the filters
filters = json.loads(live_query.filters)
# Translate the filters
for filter in filters.values():
for field in filter.keys():
value = filter.pop(field)
filter[mapping.get(field, field)] = value
return filters, lq_fields
def _service_dict_from_mongo_item(mongo_item):
"""Create a dict from a mongodb item."""
mappings = [
('last_chk', 'last_check', int),
('last_state_change', 'last_state_change', int),
('output', 'plugin_output', str),
('problem_has_been_acknowledged', 'acknowledged', bool),
]
for field in mappings:
value = mongo_item.pop(field[0], None)
if value is not None:
mongo_item[field[1]] = field[2](value)
return mongo_item

38
surveil/api/handlers/status/mongodb_query.py Normal file
View File

@ -0,0 +1,38 @@
# Copyright 2015 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def build_mongodb_query(lq_filters, lq_fields):
# Build the query
query = {}
for filter_name, filter_data in lq_filters.items():
for field, values in filter_data.items():
query[field] = {
_get_mongo_filter(filter_name): values
}
# Build the required fields
fields = {}
for field in lq_fields:
fields[field] = 1
return query, fields
def _get_mongo_filter(livequery_filter):
filters = {
"is": "$in",
"isnot": "$nin"
}
return filters[livequery_filter]

85
surveil/tests/api/controllers/v2/status/test_hosts.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import json
import httpretty
@ -101,6 +102,30 @@ class TestStatusHosts(functionalTest.FunctionalTest):
]
})
self.services = [
{
"host_name": 'Webserver',
"service_description": 'Apache',
"description": 'Serves Stuff',
"state": 'OK',
"last_check": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'HTTP OK - GOT NICE RESPONSE'
},
{
"host_name": 'someserver',
"service_description": 'servicesomething',
"description": 'Serves lots of Stuff',
"state": 'OK',
"last_check": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'Hi there'
},
]
self.mongoconnection.shinken_live.services.insert(
copy.deepcopy(self.services)
)
@httpretty.activate
def test_get_all_hosts(self):
httpretty.register_uri(httpretty.GET,
@ -187,7 +212,7 @@ class TestStatusHosts(functionalTest.FunctionalTest):
body=influxdb_response)
query = {
'fields': json.dumps(['host_name', 'last_check']),
'fields': ['host_name', 'last_check'],
'filters': json.dumps({
"isnot": {
"host_name": ['localhost'],
@ -264,70 +289,16 @@ class TestStatusHosts(functionalTest.FunctionalTest):
"LIMIT 1"]
)
@httpretty.activate
def test_get_specific_host_service(self):
influx_response = json.dumps(
{"results": [
{"series": [
{"name": "SERVICE_STATE",
"tags": {"host_name": "ws-arbiter",
"service_description": "check-ws-arbiter"},
"columns": ["time",
"acknowledged",
"last_check",
"last_state_change",
"output",
"state",
"state_type"],
"values":[
["2015-04-23T21:12:11Z",
0,
1.429823531e+09,
1.42982353221745e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD"],
["2015-04-23T21:17:11Z",
0,
1.429823831e+09,
1.42982353221745e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD"],
["2015-04-23T21:22:10Z",
0,
1.42982413e+09,
1.42982353221745e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD"]]}]}]}
)
httpretty.register_uri(httpretty.GET,
"http://influxdb:8086/query",
body=influx_response)
response = self.get(
"/v2/status/hosts/ws-arbiter/services/check-ws-arbiter"
"/v2/status/hosts/someserver/services/servicesomething"
)
expected = {'description': 'check-ws-arbiter',
'last_state_change': 1429823532,
'acknowledged': 0,
'plugin_output': ('TCP OK - 0.000 second '
'response time on port 7760'),
'last_check': 1429823531,
'state': 0,
'state': 'OK',
'host_name': 'ws-arbiter',
'service_description': 'check-ws-arbiter'}
self.assertItemsEqual(json.loads(response.body), expected)
self.assertEqual(
httpretty.last_request().querystring['q'],
["SELECT * from LIVE_SERVICE_STATE "
"WHERE host_name='ws-arbiter' "
"AND service_description='check-ws-arbiter' "
"GROUP BY * "
"ORDER BY time DESC "
"LIMIT 1"]
)

175
surveil/tests/api/controllers/v2/status/test_services.py
View File

@ -12,10 +12,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import json
import httpretty
from surveil.tests.api import functionalTest
@ -23,140 +22,70 @@ class TestStatusServices(functionalTest.FunctionalTest):
def setUp(self):
super(TestStatusServices, self).setUp()
self.influxdb_response = json.dumps({
"results": [
{"series": [
{"name": "LIVE_SERVICE_STATE",
"tags": {"host_name": "test_keystone",
"service_description":
"Check KeyStone service."},
"columns": [
"time",
"last_check",
"last_state_change",
"output",
"state",
"state_type",
"acknowledged"
],
"values":[
["2015-04-19T18:20:34Z",
1.429467634e+09,
1.429467636632134e+09,
("There was no suitable "
"authentication url for this request"),
3,
"SOFT",
0]
]},
{"name": "LIVE_SERVICE_STATE",
"tags": {"host_name": "ws-arbiter",
"service_description": "check-ws-arbiter"},
"columns": [
"time",
"last_check",
"last_state_change",
"output",
"state",
"state_type",
"acknowledged"
],
"values":[
["2015-04-19T18:20:33Z",
1.429467633e+09,
1.429467635629833e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD",
0]
]}
]}
]
})
self.services = [
{
"host_name": 'Webserver',
"service_description": 'Apache',
"description": 'Serves Stuff',
"state": 'OK',
"last_chk": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'HTTP OK - GOT NICE RESPONSE',
"problem_has_been_acknowledged": True,
},
{
"host_name": 'someserver',
"service_description": 'servicesomething',
"description": 'Serves lots of Stuff',
"state": 'UNKNOWN',
"last_chk": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'Hi there',
"problem_has_been_acknowledged": False,
},
]
self.mongoconnection.shinken_live.services.insert(
copy.deepcopy(self.services)
)
@httpretty.activate
def test_get_all_services(self):
httpretty.register_uri(httpretty.GET,
"http://influxdb:8086/query",
body=self.influxdb_response)
response = self.get("/v2/status/services")
expected = [
{'description': 'Check KeyStone service.',
'last_state_change': 1429467636,
'plugin_output':
'There was no suitable authentication url for this request',
'last_check': 1429467634,
'state': 3,
"acknowledged": 0,
'host_name': 'test_keystone',
'service_description': 'Check KeyStone service.'},
{'description': 'check-ws-arbiter',
'last_state_change': 1429467635,
'plugin_output':
'TCP OK - 0.000 second response time on port 7760',
'last_check': 1429467633,
'state': 0,
"acknowledged": 0,
'host_name': 'ws-arbiter',
'service_description': 'check-ws-arbiter'}
{
"host_name": 'Webserver',
"service_description": 'Apache',
"description": 'Serves Stuff',
"state": 'OK',
"last_check": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'HTTP OK - GOT NICE RESPONSE',
'acknowledged': True,
},
{
"host_name": 'someserver',
"service_description": 'servicesomething',
"description": 'Serves lots of Stuff',
"state": 'UNKNOWN',
"last_check": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'Hi there',
'acknowledged': False,
},
]
self.assertEqual(json.loads(response.body), expected)
self.assertItemsEqual(json.loads(response.body), expected)
self.assertEqual(
httpretty.last_request().querystring['q'],
["SELECT * FROM LIVE_SERVICE_STATE GROUP BY host_name,"
" service_description "
"ORDER BY time DESC "
"LIMIT 1"]
)
@httpretty.activate
def test_query_services(self):
influxdb_response = json.dumps({
"results": [
{"series": [
{"name": "LIVE_SERVICE_STATE",
"tags": {"host_name": "test_keystone",
"service_description":
"Check KeyStone service."},
"columns": [
"time",
"last_check",
"last_state_change",
"output",
"state",
"state_type",
"acknowledged"
],
"values":[
["2015-04-19T18:20:34Z",
1.429467634e+09,
1.429467636632134e+09,
("There was no suitable "
"authentication url for this request"),
3,
"SOFT",
0]
]}
]}
]
})
httpretty.register_uri(httpretty.GET,
"http://influxdb:8086/query",
body=influxdb_response)
query = {
'fields': json.dumps(['host_name', 'service_description']),
'fields': ['host_name', 'service_description'],
'filters': json.dumps({
"isnot": {
"host_name": ['ws-arbiter'],
},
"is": {
"service_description": ["Check KeyStone service."]
"service_description": ["Apache"],
"acknowledged": [True],
}
})
}
@ -164,8 +93,8 @@ class TestStatusServices(functionalTest.FunctionalTest):
response = self.post_json("/v2/status/services", params=query)
expected = [
{'host_name': 'test_keystone',
'service_description': 'Check KeyStone service.'}
{'host_name': 'Webserver',
'service_description': 'Apache'}
]
self.assertEqual(json.loads(response.body), expected)

50
surveil/tests/api/handlers/live/test_fields_filter.py Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2015 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from surveil.api.datamodel.status import live_query
from surveil.api.handlers.status import fields_filter
from surveil.tests import base
class FieldsFilterTest(base.BaseTestCase):
def test_filter_fields(self):
items = [
{"description": "test_keystone",
"last_state_change": 1429400986,
"plugin_output": "OK - 127.0.0.1: rta 0.045ms, lost 0%",
"last_check": 1429400984, "state": 'OK',
"host_name": "test_keystone"},
]
query = live_query.LiveQuery(
fields=['host_name', 'last_check'],
filters=json.dumps({
"isnot": {
"state": [0, 1],
"description": ["test_keystone"]
}
})
)
result = fields_filter.filter_fields(
items,
query
)
expected = [{"last_check": 1429400984, "host_name": "test_keystone"}]
self.assertItemsEqual(result, expected)

31
surveil/tests/api/handlers/live/test_influxdb_query.py
View File

@ -21,33 +21,6 @@ from surveil.tests import base
class LiveQueryFilterTest(base.BaseTestCase):
def test_filter_fields(self):
items = [
{"description": "test_keystone",
"last_state_change": 1429400986,
"plugin_output": "OK - 127.0.0.1: rta 0.045ms, lost 0%",
"last_check": 1429400984, "state": 2,
"host_name": "test_keystone"},
]
query = live_query.LiveQuery(
fields=json.dumps(['host_name', 'last_check']),
filters=json.dumps({
"isnot": {
"state": [0, 1],
"description": ["test_keystone"]
}
})
)
result = influxdb_query.filter_fields(
items,
query
)
expected = [{"last_check": 1429400984, "host_name": "test_keystone"}]
self.assertItemsEqual(result, expected)
def test_build_where_clause(self):
filters = {
"is": {
@ -77,7 +50,7 @@ class LiveQueryFilterTest(base.BaseTestCase):
def test_build_influx_query(self):
query = live_query.LiveQuery(
fields=json.dumps(['host_name', 'last_check']),
fields=['host_name', 'last_check'],
filters=json.dumps({}),
)
measurement = 'ALERT'
@ -95,7 +68,7 @@ class LiveQueryFilterTest(base.BaseTestCase):
def test_build_influx_query_orderby(self):
query = live_query.LiveQuery(
fields=json.dumps(['host_name', 'last_check']),
fields=['host_name', 'last_check'],
filters=json.dumps({}),
)
measurement = 'ALERT'

62
surveil/tests/api/handlers/live/test_mongodb_query.py Normal file
View File

@ -0,0 +1,62 @@
# Copyright 2015 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from surveil.api.datamodel.status import live_query
from surveil.api.handlers.status import live_service_handler
from surveil.api.handlers.status import mongodb_query
from surveil.tests import base
class MongoDBQueryTest(base.BaseTestCase):
def test_build_mongo_query(self):
query = live_query.LiveQuery(
fields=['host_name', 'last_check'],
filters=json.dumps({
"isnot": {
"state": [0, 1],
"last_check": ["test_keystone"]
}
})
)
lq_filters, lq_fields = live_service_handler._translate_live_query(
query
)
self.assertEqual(
lq_fields, ['host_name', 'last_chk']
)
self.assertEqual(lq_filters,
{'isnot': {'state': [0, 1],
'last_chk': ['test_keystone']}})
query, fields = mongodb_query.build_mongodb_query(lq_filters,
lq_fields)
expected_query = {
"state": {"$nin": [0, 1]},
"last_chk": {"$nin": ["test_keystone"]}
}
expected_fields = {
"host_name": 1,
"last_chk": 1
}
self.assertEqual(query, expected_query)
self.assertEqual(fields, expected_fields)