Service status API now uses live_shinken

Change-Id: I8a0b986e3ee3bc926b8daa9ea818bc7a2f7f679e
This commit is contained in:
aviau 2015-05-11 11:01:40 -04:00
parent 39f1a88a9a
commit cfc49075d3
12 changed files with 350 additions and 285 deletions

View File

@ -26,13 +26,13 @@ class LiveQuery(types.Base):
filters = wsme.wsattr(wtypes.text, mandatory=True) filters = wsme.wsattr(wtypes.text, mandatory=True)
"The filter expression encoded in json." "The filter expression encoded in json."
fields = wsme.wsattr(wtypes.text, mandatory=False) fields = wsme.wsattr([wtypes.text], mandatory=False)
"List of fields to include in the response." "List of fields to include in the response."
@classmethod @classmethod
def sample(cls): def sample(cls):
return cls( return cls(
fields=json.dumps(['host_name', 'last_check']), fields=['host_name', 'last_check'],
filters=json.dumps({ filters=json.dumps({
"isnot": { "isnot": {
"state": ["0", "1"], "state": ["0", "1"],

View File

@ -28,10 +28,10 @@ class LiveService(types.Base):
description = wsme.wsattr(wtypes.text, mandatory=False) description = wsme.wsattr(wtypes.text, mandatory=False)
"""The description of the sevice""" """The description of the sevice"""
state = wsme.wsattr(int, mandatory=False) state = wsme.wsattr(wtypes.text, mandatory=False)
"""The current state of the service""" """The current state of the service"""
acknowledged = wsme.wsattr(int, mandatory=False) acknowledged = wsme.wsattr(bool, mandatory=False)
"""Wether or not the problem, if any, has been acknowledged""" """Wether or not the problem, if any, has been acknowledged"""
last_check = wsme.wsattr(int, mandatory=False) last_check = wsme.wsattr(int, mandatory=False)
@ -47,10 +47,11 @@ class LiveService(types.Base):
def sample(cls): def sample(cls):
return cls( return cls(
host_name='Webserver', host_name='Webserver',
service_name='Apache', service_description='Apache',
description='Serves Stuff', description='Serves Stuff',
state=0, state='OK',
last_check=1429220785, last_check=1429220785,
last_state_change=1429220785, last_state_change=1429220785,
plugin_output='HTTP OK - GOT NICE RESPONSE' plugin_output='HTTP OK - GOT NICE RESPONSE',
acknowledged=True,
) )

View File

@ -0,0 +1,32 @@
# Copyright 2014 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import wsme
def filter_fields(item_list, live_query):
"""Takes unwanted keys out of a dict depending on a live_query."""
filtered_items = []
if live_query.fields != wsme.Unset:
fields = live_query.fields
for item in item_list:
filtered_item = {}
for field in fields:
filtered_item[field] = item.get(field, None)
filtered_items.append(filtered_item)
else:
filtered_items = item_list
return filtered_items

View File

@ -14,24 +14,6 @@
import json import json
import wsme
def filter_fields(item_list, live_query):
filtered_items = []
if live_query.fields != wsme.Unset:
fields = json.loads(live_query.fields)
for item in item_list:
filtered_item = {}
for field in fields:
filtered_item[field] = item[field]
filtered_items.append(filtered_item)
else:
filtered_items = item_list
return filtered_items
def build_influxdb_query(live_query, def build_influxdb_query(live_query,
measurement, measurement,

View File

@ -17,6 +17,7 @@ import json
from surveil.api.datamodel.status import live_host from surveil.api.datamodel.status import live_host
from surveil.api.handlers import handler from surveil.api.handlers import handler
from surveil.api.handlers.status import fields_filter
from surveil.api.handlers.status import influxdb_query from surveil.api.handlers.status import influxdb_query
@ -57,7 +58,7 @@ class HostHandler(handler.Handler):
host_dicts.append(host_dict) host_dicts.append(host_dict)
if live_query: if live_query:
host_dicts = influxdb_query.filter_fields( host_dicts = fields_filter.filter_fields(
host_dicts, host_dicts,
live_query live_query
) )

View File

@ -12,56 +12,48 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
from __future__ import print_function import json
from surveil.api.datamodel.status import live_service from surveil.api.datamodel.status import live_service
from surveil.api.handlers import handler from surveil.api.handlers import handler
from surveil.api.handlers.status import influxdb_query from surveil.api.handlers.status import mongodb_query
import wsme
class ServiceHandler(handler.Handler): class ServiceHandler(handler.Handler):
"""Fulfills a request on live services.""" """Fulfills a request on live services."""
def get(self, host_name, service_name): def get(self, host_name, service_description):
"""Return a specific service.""" """Return a specific service."""
cli = self.request.influxdb_client mongo_s = self.request.mongo_connection.shinken_live.services.find_one(
query = ("SELECT * from LIVE_SERVICE_STATE " {"host_name": host_name,
"WHERE host_name='%s' " "service_description": service_description},
"AND service_description='%s' "
"GROUP BY * "
"ORDER BY time DESC "
"LIMIT 1") % (host_name, service_name)
response = cli.query(query)
host = live_service.LiveService(
**self._service_dict_from_influx_item(response.items()[0])
) )
return host return live_service.LiveService(**mongo_s)
def get_all(self, live_query=None): def get_all(self, live_query=None):
"""Return all live services.""" """Return all live services."""
cli = self.request.influxdb_client
query = influxdb_query.build_influxdb_query(
live_query,
'LIVE_SERVICE_STATE',
group_by=['host_name', 'service_description'],
order_by=['time DESC'],
limit=1
)
response = cli.query(query)
service_dicts = []
for item in response.items():
service_dict = self._service_dict_from_influx_item(item)
service_dicts.append(service_dict)
if live_query: if live_query:
service_dicts = influxdb_query.filter_fields( lq_filters, lq_fields = _translate_live_query(live_query)
service_dicts, else:
live_query lq_filters = {}
) lq_fields = {}
query, fields = mongodb_query.build_mongodb_query(lq_filters,
lq_fields)
if fields != {}:
mongo_dicts = (self.request.mongo_connection.
shinken_live.services.find(query, fields))
else:
mongo_dicts = (self.request.mongo_connection.
shinken_live.services.find(query))
service_dicts = [
_service_dict_from_mongo_item(s) for s in mongo_dicts
]
services = [] services = []
for service_dict in service_dicts: for service_dict in service_dicts:
@ -70,20 +62,54 @@ class ServiceHandler(handler.Handler):
return services return services
def _service_dict_from_influx_item(self, item):
tags = item[0][1]
points = item[1]
first_point = next(points)
service_dict = { def _translate_live_query(live_query):
"service_description": tags['service_description'], """Translate field names in a live query so that they match mongodb."""
"host_name": tags['host_name'],
"description": tags['service_description'],
"state": first_point['state'],
"acknowledged": int(first_point['acknowledged']),
"last_check": int(first_point['last_check']),
"last_state_change": int(first_point['last_state_change']),
"plugin_output": first_point['output']
}
return service_dict # Mappings
mapping = {
"last_check": "last_chk",
"description": "service_description",
"plugin_output": "output",
"acknowledged": "problem_has_been_acknowledged",
}
# Load the fields
if live_query.fields != wsme.Unset:
fields = live_query.fields
else:
fields = []
# Translate the fields
lq_fields = []
for field in fields:
lq_fields.append(mapping.get(field, field))
# Load the filters
filters = json.loads(live_query.filters)
# Translate the filters
for filter in filters.values():
for field in filter.keys():
value = filter.pop(field)
filter[mapping.get(field, field)] = value
return filters, lq_fields
def _service_dict_from_mongo_item(mongo_item):
"""Create a dict from a mongodb item."""
mappings = [
('last_chk', 'last_check', int),
('last_state_change', 'last_state_change', int),
('output', 'plugin_output', str),
('problem_has_been_acknowledged', 'acknowledged', bool),
]
for field in mappings:
value = mongo_item.pop(field[0], None)
if value is not None:
mongo_item[field[1]] = field[2](value)
return mongo_item

View File

@ -0,0 +1,38 @@
# Copyright 2015 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def build_mongodb_query(lq_filters, lq_fields):
# Build the query
query = {}
for filter_name, filter_data in lq_filters.items():
for field, values in filter_data.items():
query[field] = {
_get_mongo_filter(filter_name): values
}
# Build the required fields
fields = {}
for field in lq_fields:
fields[field] = 1
return query, fields
def _get_mongo_filter(livequery_filter):
filters = {
"is": "$in",
"isnot": "$nin"
}
return filters[livequery_filter]

View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import copy
import json import json
import httpretty import httpretty
@ -101,6 +102,30 @@ class TestStatusHosts(functionalTest.FunctionalTest):
] ]
}) })
self.services = [
{
"host_name": 'Webserver',
"service_description": 'Apache',
"description": 'Serves Stuff',
"state": 'OK',
"last_check": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'HTTP OK - GOT NICE RESPONSE'
},
{
"host_name": 'someserver',
"service_description": 'servicesomething',
"description": 'Serves lots of Stuff',
"state": 'OK',
"last_check": 1429220785,
"last_state_change": 1429220785,
"plugin_output": 'Hi there'
},
]
self.mongoconnection.shinken_live.services.insert(
copy.deepcopy(self.services)
)
@httpretty.activate @httpretty.activate
def test_get_all_hosts(self): def test_get_all_hosts(self):
httpretty.register_uri(httpretty.GET, httpretty.register_uri(httpretty.GET,
@ -187,7 +212,7 @@ class TestStatusHosts(functionalTest.FunctionalTest):
body=influxdb_response) body=influxdb_response)
query = { query = {
'fields': json.dumps(['host_name', 'last_check']), 'fields': ['host_name', 'last_check'],
'filters': json.dumps({ 'filters': json.dumps({
"isnot": { "isnot": {
"host_name": ['localhost'], "host_name": ['localhost'],
@ -264,70 +289,16 @@ class TestStatusHosts(functionalTest.FunctionalTest):
"LIMIT 1"] "LIMIT 1"]
) )
@httpretty.activate
def test_get_specific_host_service(self): def test_get_specific_host_service(self):
influx_response = json.dumps(
{"results": [
{"series": [
{"name": "SERVICE_STATE",
"tags": {"host_name": "ws-arbiter",
"service_description": "check-ws-arbiter"},
"columns": ["time",
"acknowledged",
"last_check",
"last_state_change",
"output",
"state",
"state_type"],
"values":[
["2015-04-23T21:12:11Z",
0,
1.429823531e+09,
1.42982353221745e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD"],
["2015-04-23T21:17:11Z",
0,
1.429823831e+09,
1.42982353221745e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD"],
["2015-04-23T21:22:10Z",
0,
1.42982413e+09,
1.42982353221745e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD"]]}]}]}
)
httpretty.register_uri(httpretty.GET,
"http://influxdb:8086/query",
body=influx_response)
response = self.get( response = self.get(
"/v2/status/hosts/ws-arbiter/services/check-ws-arbiter" "/v2/status/hosts/someserver/services/servicesomething"
) )
expected = {'description': 'check-ws-arbiter', expected = {'description': 'check-ws-arbiter',
'last_state_change': 1429823532, 'last_state_change': 1429823532,
'acknowledged': 0,
'plugin_output': ('TCP OK - 0.000 second ' 'plugin_output': ('TCP OK - 0.000 second '
'response time on port 7760'), 'response time on port 7760'),
'last_check': 1429823531, 'last_check': 1429823531,
'state': 0, 'state': 'OK',
'host_name': 'ws-arbiter', 'host_name': 'ws-arbiter',
'service_description': 'check-ws-arbiter'} 'service_description': 'check-ws-arbiter'}
self.assertItemsEqual(json.loads(response.body), expected) self.assertItemsEqual(json.loads(response.body), expected)
self.assertEqual(
httpretty.last_request().querystring['q'],
["SELECT * from LIVE_SERVICE_STATE "
"WHERE host_name='ws-arbiter' "
"AND service_description='check-ws-arbiter' "
"GROUP BY * "
"ORDER BY time DESC "
"LIMIT 1"]
)

View File

@ -12,10 +12,9 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import copy
import json import json
import httpretty
from surveil.tests.api import functionalTest from surveil.tests.api import functionalTest
@ -23,140 +22,70 @@ class TestStatusServices(functionalTest.FunctionalTest):
def setUp(self): def setUp(self):
super(TestStatusServices, self).setUp() super(TestStatusServices, self).setUp()
self.influxdb_response = json.dumps({ self.services = [
"results": [ {
{"series": [ "host_name": 'Webserver',
{"name": "LIVE_SERVICE_STATE", "service_description": 'Apache',
"tags": {"host_name": "test_keystone", "description": 'Serves Stuff',
"service_description": "state": 'OK',
"Check KeyStone service."}, "last_chk": 1429220785,
"columns": [ "last_state_change": 1429220785,
"time", "plugin_output": 'HTTP OK - GOT NICE RESPONSE',
"last_check", "problem_has_been_acknowledged": True,
"last_state_change", },
"output", {
"state", "host_name": 'someserver',
"state_type", "service_description": 'servicesomething',
"acknowledged" "description": 'Serves lots of Stuff',
], "state": 'UNKNOWN',
"values":[ "last_chk": 1429220785,
["2015-04-19T18:20:34Z", "last_state_change": 1429220785,
1.429467634e+09, "plugin_output": 'Hi there',
1.429467636632134e+09, "problem_has_been_acknowledged": False,
("There was no suitable " },
"authentication url for this request"), ]
3, self.mongoconnection.shinken_live.services.insert(
"SOFT", copy.deepcopy(self.services)
0] )
]},
{"name": "LIVE_SERVICE_STATE",
"tags": {"host_name": "ws-arbiter",
"service_description": "check-ws-arbiter"},
"columns": [
"time",
"last_check",
"last_state_change",
"output",
"state",
"state_type",
"acknowledged"
],
"values":[
["2015-04-19T18:20:33Z",
1.429467633e+09,
1.429467635629833e+09,
"TCP OK - 0.000 second response time on port 7760",
0,
"HARD",
0]
]}
]}
]
})
@httpretty.activate
def test_get_all_services(self): def test_get_all_services(self):
httpretty.register_uri(httpretty.GET,
"http://influxdb:8086/query",
body=self.influxdb_response)
response = self.get("/v2/status/services") response = self.get("/v2/status/services")
expected = [ expected = [
{'description': 'Check KeyStone service.', {
'last_state_change': 1429467636, "host_name": 'Webserver',
'plugin_output': "service_description": 'Apache',
'There was no suitable authentication url for this request', "description": 'Serves Stuff',
'last_check': 1429467634, "state": 'OK',
'state': 3, "last_check": 1429220785,
"acknowledged": 0, "last_state_change": 1429220785,
'host_name': 'test_keystone', "plugin_output": 'HTTP OK - GOT NICE RESPONSE',
'service_description': 'Check KeyStone service.'}, 'acknowledged': True,
{'description': 'check-ws-arbiter', },
'last_state_change': 1429467635, {
'plugin_output': "host_name": 'someserver',
'TCP OK - 0.000 second response time on port 7760', "service_description": 'servicesomething',
'last_check': 1429467633, "description": 'Serves lots of Stuff',
'state': 0, "state": 'UNKNOWN',
"acknowledged": 0, "last_check": 1429220785,
'host_name': 'ws-arbiter', "last_state_change": 1429220785,
'service_description': 'check-ws-arbiter'} "plugin_output": 'Hi there',
'acknowledged': False,
},
] ]
self.assertEqual(json.loads(response.body), expected) self.assertItemsEqual(json.loads(response.body), expected)
self.assertEqual(
httpretty.last_request().querystring['q'],
["SELECT * FROM LIVE_SERVICE_STATE GROUP BY host_name,"
" service_description "
"ORDER BY time DESC "
"LIMIT 1"]
)
@httpretty.activate
def test_query_services(self): def test_query_services(self):
influxdb_response = json.dumps({
"results": [
{"series": [
{"name": "LIVE_SERVICE_STATE",
"tags": {"host_name": "test_keystone",
"service_description":
"Check KeyStone service."},
"columns": [
"time",
"last_check",
"last_state_change",
"output",
"state",
"state_type",
"acknowledged"
],
"values":[
["2015-04-19T18:20:34Z",
1.429467634e+09,
1.429467636632134e+09,
("There was no suitable "
"authentication url for this request"),
3,
"SOFT",
0]
]}
]}
]
})
httpretty.register_uri(httpretty.GET,
"http://influxdb:8086/query",
body=influxdb_response)
query = { query = {
'fields': json.dumps(['host_name', 'service_description']), 'fields': ['host_name', 'service_description'],
'filters': json.dumps({ 'filters': json.dumps({
"isnot": { "isnot": {
"host_name": ['ws-arbiter'], "host_name": ['ws-arbiter'],
}, },
"is": { "is": {
"service_description": ["Check KeyStone service."] "service_description": ["Apache"],
"acknowledged": [True],
} }
}) })
} }
@ -164,8 +93,8 @@ class TestStatusServices(functionalTest.FunctionalTest):
response = self.post_json("/v2/status/services", params=query) response = self.post_json("/v2/status/services", params=query)
expected = [ expected = [
{'host_name': 'test_keystone', {'host_name': 'Webserver',
'service_description': 'Check KeyStone service.'} 'service_description': 'Apache'}
] ]
self.assertEqual(json.loads(response.body), expected) self.assertEqual(json.loads(response.body), expected)

View File

@ -0,0 +1,50 @@
# Copyright 2015 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from surveil.api.datamodel.status import live_query
from surveil.api.handlers.status import fields_filter
from surveil.tests import base
class FieldsFilterTest(base.BaseTestCase):
def test_filter_fields(self):
items = [
{"description": "test_keystone",
"last_state_change": 1429400986,
"plugin_output": "OK - 127.0.0.1: rta 0.045ms, lost 0%",
"last_check": 1429400984, "state": 'OK',
"host_name": "test_keystone"},
]
query = live_query.LiveQuery(
fields=['host_name', 'last_check'],
filters=json.dumps({
"isnot": {
"state": [0, 1],
"description": ["test_keystone"]
}
})
)
result = fields_filter.filter_fields(
items,
query
)
expected = [{"last_check": 1429400984, "host_name": "test_keystone"}]
self.assertItemsEqual(result, expected)

View File

@ -21,33 +21,6 @@ from surveil.tests import base
class LiveQueryFilterTest(base.BaseTestCase): class LiveQueryFilterTest(base.BaseTestCase):
def test_filter_fields(self):
items = [
{"description": "test_keystone",
"last_state_change": 1429400986,
"plugin_output": "OK - 127.0.0.1: rta 0.045ms, lost 0%",
"last_check": 1429400984, "state": 2,
"host_name": "test_keystone"},
]
query = live_query.LiveQuery(
fields=json.dumps(['host_name', 'last_check']),
filters=json.dumps({
"isnot": {
"state": [0, 1],
"description": ["test_keystone"]
}
})
)
result = influxdb_query.filter_fields(
items,
query
)
expected = [{"last_check": 1429400984, "host_name": "test_keystone"}]
self.assertItemsEqual(result, expected)
def test_build_where_clause(self): def test_build_where_clause(self):
filters = { filters = {
"is": { "is": {
@ -77,7 +50,7 @@ class LiveQueryFilterTest(base.BaseTestCase):
def test_build_influx_query(self): def test_build_influx_query(self):
query = live_query.LiveQuery( query = live_query.LiveQuery(
fields=json.dumps(['host_name', 'last_check']), fields=['host_name', 'last_check'],
filters=json.dumps({}), filters=json.dumps({}),
) )
measurement = 'ALERT' measurement = 'ALERT'
@ -95,7 +68,7 @@ class LiveQueryFilterTest(base.BaseTestCase):
def test_build_influx_query_orderby(self): def test_build_influx_query_orderby(self):
query = live_query.LiveQuery( query = live_query.LiveQuery(
fields=json.dumps(['host_name', 'last_check']), fields=['host_name', 'last_check'],
filters=json.dumps({}), filters=json.dumps({}),
) )
measurement = 'ALERT' measurement = 'ALERT'

View File

@ -0,0 +1,62 @@
# Copyright 2015 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from surveil.api.datamodel.status import live_query
from surveil.api.handlers.status import live_service_handler
from surveil.api.handlers.status import mongodb_query
from surveil.tests import base
class MongoDBQueryTest(base.BaseTestCase):
def test_build_mongo_query(self):
query = live_query.LiveQuery(
fields=['host_name', 'last_check'],
filters=json.dumps({
"isnot": {
"state": [0, 1],
"last_check": ["test_keystone"]
}
})
)
lq_filters, lq_fields = live_service_handler._translate_live_query(
query
)
self.assertEqual(
lq_fields, ['host_name', 'last_chk']
)
self.assertEqual(lq_filters,
{'isnot': {'state': [0, 1],
'last_chk': ['test_keystone']}})
query, fields = mongodb_query.build_mongodb_query(lq_filters,
lq_fields)
expected_query = {
"state": {"$nin": [0, 1]},
"last_chk": {"$nin": ["test_keystone"]}
}
expected_fields = {
"host_name": 1,
"last_chk": 1
}
self.assertEqual(query, expected_query)
self.assertEqual(fields, expected_fields)