x/surveil
Code Issues Proposed changes

Added MongoDB objects validation

Browse Source 
Change-Id: Id05afa718d9c2b0c8e1eb09645a3b5a3ae489e7c
This commit is contained in:
aviau 2015-07-27 16:47:47 -04:00
parent 628f1baa56
commit 9a9cd3f50f
20 changed files with 273 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
surveil/api/storage/mongodb/config/businessimpactmodulation.py
View File

@ -17,6 +17,6 @@ import mongoengine
class BusinessImpactModulation(mongoengine.Document):
meta = {'collection': 'businessimpactmodulations'}
business_impact_modulation_name = mongoengine.StringField()
business_impact_modulation_name = mongoengine.StringField(unique=True)
business_impact = mongoengine.IntField()
modulation_period = mongoengine.StringField()

2
surveil/api/storage/mongodb/config/checkmodulation.py
View File

@ -17,6 +17,6 @@ import mongoengine
class CheckModulation(mongoengine.Document):
meta = {'collection': 'checkmodulations'}
checkmodulation_name = mongoengine.StringField()
checkmodulation_name = mongoengine.StringField(unique=True)
check_command = mongoengine.StringField()
check_period = mongoengine.StringField()

2
surveil/api/storage/mongodb/config/command.py
View File

@ -20,6 +20,6 @@ class Command(mongoengine.Document):
'collection': 'commands',
'strict': False
}
command_name = mongoengine.StringField()
command_name = mongoengine.StringField(unique=True)
command_line = mongoengine.StringField()
module_type = mongoengine.StringField()

2
surveil/api/storage/mongodb/config/contact.py
View File

@ -17,7 +17,7 @@ import mongoengine
class Contact(mongoengine.Document):
meta = {'collection': 'contacts'}
contact_name = mongoengine.StringField()
contact_name = mongoengine.StringField(unique=True)
host_notifications_enabled = mongoengine.StringField()
service_notifications_enabled = mongoengine.StringField()
host_notification_period = mongoengine.StringField()

8
surveil/api/storage/mongodb/config/contactgroup.py
View File

@ -14,10 +14,14 @@
import mongoengine
from surveil.api.storage.mongodb.config import contact
from surveil.api.storage.mongodb import foreign_key_field
class ContactGroup(mongoengine.Document):
meta = {'collection': 'contactgroups'}
contactgroup_name = mongoengine.StringField()
members = mongoengine.ListField()
contactgroup_name = mongoengine.StringField(unique=True)
members = foreign_key_field.ForeignKeyListField(contact.Contact,
['contact_name'])
alias = mongoengine.StringField()
contactgroup_members = mongoengine.ListField()

6
surveil/api/storage/mongodb/config/hostgroup.py
View File

@ -14,11 +14,15 @@
import mongoengine
from surveil.api.storage.mongodb.config import host
from surveil.api.storage.mongodb import foreign_key_field
class HostGroup(mongoengine.Document):
meta = {'collection': 'hostgroups'}
hostgroup_name = mongoengine.StringField(unique=True)
members = mongoengine.ListField()
members = foreign_key_field.ForeignKeyListField(host.Host,
['host_name', 'name'])
alias = mongoengine.StringField()
hostgroup_members = mongoengine.ListField()
notes = mongoengine.StringField()

2
surveil/api/storage/mongodb/config/macromodulation.py
View File

@ -17,6 +17,6 @@ import mongoengine
class MacroModulation(mongoengine.Document):
meta = {'collection': 'macromodulations'}
macromodulation_name = mongoengine.StringField()
macromodulation_name = mongoengine.StringField(unique=True)
modulation_period = mongoengine.StringField()
macros = mongoengine.DictField()

2
surveil/api/storage/mongodb/config/notificationway.py
View File

@ -17,7 +17,7 @@ import mongoengine
class NotificationWays(mongoengine.Document):
meta = {'collection': 'notificationways'}
notificationway_name = mongoengine.StringField()
notificationway_name = mongoengine.StringField(unique=True)
host_notification_period = mongoengine.StringField()
service_notification_period = mongoengine.StringField()
host_notification_options = mongoengine.ListField()

6
surveil/api/storage/mongodb/config/service.py
View File

@ -14,13 +14,17 @@
import mongoengine
from surveil.api.storage.mongodb.config import host
from surveil.api.storage.mongodb import foreign_key_field
class Service(mongoengine.Document):
meta = {
'collection': 'services',
'strict': False
}
host_name = mongoengine.ListField()
host_name = foreign_key_field.ForeignKeyListField(host.Host,
['host_name', 'name'])
service_description = mongoengine.StringField()
contacts = mongoengine.ListField()
check_command = mongoengine.StringField()

8
surveil/api/storage/mongodb/config/servicegroup.py
View File

@ -14,11 +14,17 @@
import mongoengine
from surveil.api.storage.mongodb.config import service
from surveil.api.storage.mongodb import foreign_key_field
class ServiceGroup(mongoengine.Document):
meta = {'collection': 'servicegroups'}
servicegroup_name = mongoengine.StringField(unique=True)
members = mongoengine.ListField()
members = foreign_key_field.ForeignKeyListField(
service.Service,
['service_description', 'name']
)
alias = mongoengine.StringField()
servicegroup_members = mongoengine.ListField()
notes = mongoengine.StringField()

71
surveil/api/storage/mongodb/foreign_key_field.py Normal file
View File

@ -0,0 +1,71 @@
# Copyright 2015 - Savoir-Faire Linux inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mongoengine
def validate_refering_object_exists(referring_object, field_name, value):
return referring_object.objects(**{field_name: value}).count() > 0
class ForeignKeyStringField(mongoengine.StringField):
def __init__(self, referring_object,
possible_field_names,
*args,
**kwargs):
super(ForeignKeyStringField, self).__init__(*args, **kwargs)
self.refering_object = referring_object
self.possible_field_names = possible_field_names
def validate(self, value):
mongoengine.StringField.validate(self, value)
for field in self.possible_field_names:
if validate_refering_object_exists(self.refering_object,
field,
value) is True:
return
self.error(
"Could not find matching %s" % value
)
class ForeignKeyListField(mongoengine.ListField):
def __init__(self, referring_object,
possible_field_names,
*args,
**kwargs):
super(ForeignKeyListField, self).__init__(*args, **kwargs)
self.refering_object = referring_object
self.possible_field_names = possible_field_names
def validate(self, value):
mongoengine.ListField.validate(self, value)
for key in value:
found_matching = False
for field in self.possible_field_names:
if validate_refering_object_exists(
self.refering_object,
field,
key) is True:
found_matching = True
break
if found_matching is False:
self.error(
"Could not find matching %s" % key
)

2
surveil/cmd/pack_upload.py
View File

@ -58,7 +58,7 @@ def upload_pack(pack_dir, client):
surveil_config = surveil_from_nagios.load_config(pack_dir)
config_manager = client.config
for object_type, objects in surveil_config.items():
for object_type, objects in surveil_config:
object_manager = getattr(config_manager, object_type)
for object in objects:
object_manager.create(**object)

75
surveil/cmd/surveil_from_nagios.py
View File

@ -15,6 +15,7 @@
from __future__ import print_function
import argparse
import collections
import fnmatch
import json
import os
@ -73,9 +74,9 @@ def load_config(path):
else:
cfg_files = [path]
nagios_config = {}
nagios_config = collections.OrderedDict()
for cfg_file in cfg_files:
for cfg_file in sorted(cfg_files):
# Open the file
f = open(cfg_file, 'r')
config_string = f.read()
@ -93,12 +94,14 @@ def load_config(path):
surveil_config = _transform_config(nagios_config)
return surveil_config
sorted_surveil_config = _sort_config(surveil_config)
return sorted_surveil_config
def _load_nagios_config(config_string):
"""Given a nagios configuration string, returns a python dict"""
config = {}
config = collections.OrderedDict()
# Find all config objects
config_objects = re.finditer(
@ -132,7 +135,7 @@ def _load_nagios_config(config_string):
def _transform_config(nagios_config):
"""Given a nagios config dict, returns surveil configuration"""
transformed_config = {}
transformed_config = collections.OrderedDict()
for object_type, objects in nagios_config.items():
for config_object in objects:
@ -158,7 +161,7 @@ def _transform_config(nagios_config):
def _transform_property_types(config_object, object_type):
transformed_object = {}
transformed_object = collections.OrderedDict()
datamodels = {
"businessimpactmodulations": businessimpactmodulation.BusinessImpactModulation,
@ -194,7 +197,7 @@ def _transform_property_types(config_object, object_type):
def _transform_property_names(config_object, object_type):
transformed_object = {}
transformed_object = collections.OrderedDict()
# HOSTS
if object_type in ['hosts', 'services']:
@ -220,7 +223,61 @@ def _transform_property_names(config_object, object_type):
return transformed_object
def _sort_config(surveil_config):
# Sort object types
correct_order = {
"realms": 0,
"timeperiods": 1,
"macromodulations": 2,
"commands": 3,
"checkmodulations": 4,
"businessimpactmodulations": 5,
"notificationways": 6,
"contacts": 7,
"contactgroups": 8,
"hosts": 9,
"services": 10,
"hostgroups": 11,
"servicegroups": 12,
}
sorted_object_types = sorted(surveil_config.items(),
key=lambda x: correct_order.get(x[0], 99))
sorted_config = []
# Sort objects
for item in sorted_object_types:
object_type = item[0]
objects = item[1]
if object_type in ['hosts', 'services']:
objects = _sort_objects(objects)
sorted_config.append((object_type, objects))
return sorted_config
def _sort_objects(objects):
sorted_objects = []
while len(objects) > 0:
for object in objects:
host_dependencies = object.get('use', [])
unsolved_dependencies = [
d for d in host_dependencies
if not any(o.get('name', None) == d
for o in sorted_objects)
]
if len(unsolved_dependencies) == 0:
break
sorted_objects.append(object)
objects.remove(object)
return sorted_objects
if __name__ == "__main__":
main()

20
surveil/tests/api/controllers/v2/config/test_contactgroup.py
View File

@ -27,12 +27,12 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
self.groups = [
{
'contactgroup_name': 'novell-admins',
'members': ["jdoe", "rtobert", "tzach"],
'members': ["bob", "alice"],
'contactgroup_members': []
},
{
'contactgroup_name': 'linux-adminx',
'members': ['linus', 'richard'],
'members': ['bob'],
'contactgroup_members': []
},
]
@ -40,6 +40,14 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
copy.deepcopy(self.groups)
)
self.contacts = [
{"contact_name": "bob"},
{"contact_name": "alice"},
]
self.mongoconnection.shinken.contacts.insert(
copy.deepcopy(self.contacts)
)
def test_get_all_contactgroups(self):
response = self.get('/v2/config/contactgroups')
@ -60,7 +68,7 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
def test_create_contactgroup(self):
g = contactgroup.ContactGroup(
contactgroup_name='John',
members=["marie", "bob", "joe"],
members=["bob", "alice"],
)
self.post_json('/v2/config/contactgroups', g.as_dict())
@ -85,18 +93,18 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
self.mongoconnection.shinken.contactgroups.find_one(
{'contactgroup_name': 'novell-admins'}
)['members'],
['jdoe', 'rtobert', 'tzach']
['bob', 'alice']
)
self.put_json(
'/v2/config/contactgroups/novell-admins',
{"contactgroup_name": "novell-admins",
"members": ["updated"]}
"members": ["bob"]}
)
self.assertEqual(
self.mongoconnection.shinken.contactgroups.find_one(
{'contactgroup_name': 'novell-admins'}
)['members'],
['updated']
['bob']
)

20
surveil/tests/api/controllers/v2/config/test_hostgroup.py
View File

@ -27,12 +27,12 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
self.groups = [
{
'hostgroup_name': 'novell-servers',
'members': ['netware1', 'netware2', 'netware3', 'netware4'],
'members': ['host1'],
'hostgroup_members': []
},
{
'hostgroup_name': 'otherservers',
'members': ['googul', 'sfl'],
'members': ['host1', 'host2'],
'hostgroup_members': []
},
]
@ -40,6 +40,14 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
copy.deepcopy(self.groups)
)
self.hosts = [
{"host_name": 'host1'},
{"host_name": 'host2'}
]
self.mongoconnection.shinken.hosts.insert(
copy.deepcopy(self.hosts)
)
def test_get_all_hostgroups(self):
response = self.get('/v2/config/hostgroups')
@ -60,7 +68,7 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
def test_create_hostgroup(self):
s = hostgroup.HostGroup(
hostgroup_name='John',
members=['marie', 'bob', 'joe'],
members=['host1', 'host2'],
)
self.post_json('/v2/config/hostgroups', s.as_dict())
@ -85,18 +93,18 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
self.mongoconnection.shinken.hostgroups.find_one(
{'hostgroup_name': 'novell-servers'}
)['members'],
['netware1', 'netware2', 'netware3', 'netware4']
['host1']
)
self.put_json(
'/v2/config/hostgroups/novell-servers',
{"hostgroup_name": "novell-servers",
"members": ["updated"]}
"members": ["host2"]}
)
self.assertEqual(
self.mongoconnection.shinken.hostgroups.find_one(
{'hostgroup_name': 'novell-servers'}
)['members'],
['updated']
['host2']
)

33
surveil/tests/api/controllers/v2/config/test_servicegroup.py
View File

@ -27,19 +27,13 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
self.groups = [
{
'servicegroup_name': 'dbservices',
'members': ['ms1',
'SQL Server',
'ms1',
'SQL Serverc Agent',
'ms1',
'SQL DTC'],
'members': ['service1'],
'servicegroup_members': []
},
{
'servicegroup_name': 'otherservices',
'members': ['some',
'other',
'member'],
'members': ['service1',
'service2'],
'servicegroup_members': []
},
]
@ -47,6 +41,14 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
copy.deepcopy(self.groups)
)
self.services = [
{'service_description': "service1"},
{'service_description': "service2"}
]
self.mongoconnection.shinken.services.insert(
copy.deepcopy(self.services)
)
def test_get_all_servicegroups(self):
response = self.get('/v2/config/servicegroups')
@ -67,7 +69,7 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
def test_create_servicegroup(self):
s = servicegroup.ServiceGroup(
servicegroup_name='John',
members=['marie', 'bob', 'joe'],
members=['service1'],
)
self.post_json('/v2/config/servicegroups', s.as_dict())
@ -92,23 +94,18 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
self.mongoconnection.shinken.servicegroups.find_one(
{'servicegroup_name': 'dbservices'}
)['members'],
['ms1',
'SQL Server',
'ms1',
'SQL Serverc Agent',
'ms1',
'SQL DTC']
['service1']
)
self.put_json(
'/v2/config/servicegroups/dbservices',
{"servicegroup_name": "dbservices",
"members": ["updated"]}
"members": ["service2"]}
)
self.assertEqual(
self.mongoconnection.shinken.servicegroups.find_one(
{'servicegroup_name': 'dbservices'}
)['members'],
['updated']
['service2']
)

9
surveil/tests/api/controllers/v2/config/test_services.py
View File

@ -71,6 +71,13 @@ class TestServiceController(functionalTest.FunctionalTest):
copy.deepcopy(self.services)
)
self.hosts = [
{"host_name": "sample-server1"}
]
self.mongoconnection.shinken.hosts.insert(
copy.deepcopy(self.services)
)
def test_get_all_services(self):
response = self.get('/v2/config/services')
@ -115,7 +122,7 @@ class TestServiceController(functionalTest.FunctionalTest):
def test_add_service(self):
new_service = {
"host_name": ["SOMEHOSTNAME"],
"host_name": ["sample-server1"],
"service_description": "check-new-thing",
"check_command": "check-disk!/dev/sdb1",
"max_check_attempts": 5,

4
surveil/tests/api/controllers/v2/config/test_timeperiods.py
View File

@ -79,7 +79,9 @@ class TestTimePeriodsController(functionalTest.FunctionalTest):
self.post_json('/v2/config/timeperiods', t)
self.assertIsNotNone(
self.mongoconnection.shinken.timeperiods.find_one(t)
self.mongoconnection.shinken.timeperiods.find_one(
{"timeperiod_name": 'someperiod'}
)
)
def test_delete_timeperiod(self):

11
surveil/tests/cmd/nagios_config/config.cfg
View File

@ -1,5 +1,5 @@
define host{
use generic-host
use generic-host,non-existing-thing
contact_groups admins
host_name localhost
address localhost
@ -7,6 +7,15 @@
check_interval 324
}
define host{
name generic-host
}
define host{
use generic-host
}
define service {
host_name test
}

84
surveil/tests/cmd/test_surveil_from_nagios.py
View File

@ -33,47 +33,47 @@ class TestSurveilFromNagios(base_test.BaseTestCase):
self.assert_count_equal_backport(
surveil_cfg,
{
'hosts': [
{
'custom_fields': {'_custom_yolo': 'sdfsdf'},
'contact_groups': [
'admins'
],
'use': [
'generic-host'
],
'host_name': 'localhost',
'check_interval': 324,
'address': 'localhost'}
],
'services': [
{
'host_name': [
'hai'
]
},
{
'host_name': [
'test'
]
}
],
'timeperiods': [
{
'timeperiod_name': 'workhours',
'alias': 'Normal Work Hours',
'periods':
{
'friday': '09:00-17:00',
'monday': '09:00-17:00',
'thursday': '09:00-17:00',
'tuesday': '09:00-17:00',
'wednesday': '09:00-17:00'
}
}
]
}
[
('timeperiods',
[
{
'alias': 'Normal Work Hours',
'timeperiod_name': 'workhours',
'periods': {
'tuesday': '09:00-17:00',
'friday': '09:00-17:00',
'thursday': '09:00-17:00',
'wednesday': '09:00-17:00',
'monday': '09:00-17:00'
}
}
]),
('hosts',
[
{
'name': 'generic-host',
'custom_fields': {}},
{
'use': ['generic-host'],
'custom_fields': {}},
{
'use': ['generic-host', 'non-existing-thing'],
'contact_groups': ['admins'],
'host_name': 'localhost',
'check_interval': 324,
'address': 'localhost',
'custom_fields': {
'_custom_yolo': 'sdfsdf'
}
},
]),
('services',
[
{'host_name': ['test']},
{'host_name': ['hai']}
])
]
)
def test_load_single_file(self):
@ -88,5 +88,5 @@ class TestSurveilFromNagios(base_test.BaseTestCase):
self.assertEqual(
single_file_config,
{'services': [{'host_name': ['hai']}]}
[('services', [{'host_name': ['hai']}])]
)