Added MongoDB objects validation

Change-Id: Id05afa718d9c2b0c8e1eb09645a3b5a3ae489e7c

surveil/api/storage/mongodb/config/businessimpactmodulation.py

@@ -17,6 +17,6 @@ import mongoengine
 
 class BusinessImpactModulation(mongoengine.Document):
     meta = {'collection': 'businessimpactmodulations'}
-    business_impact_modulation_name = mongoengine.StringField()
+    business_impact_modulation_name = mongoengine.StringField(unique=True)
     business_impact = mongoengine.IntField()
     modulation_period = mongoengine.StringField()

surveil/api/storage/mongodb/config/checkmodulation.py

@@ -17,6 +17,6 @@ import mongoengine
 
 class CheckModulation(mongoengine.Document):
     meta = {'collection': 'checkmodulations'}
-    checkmodulation_name = mongoengine.StringField()
+    checkmodulation_name = mongoengine.StringField(unique=True)
     check_command = mongoengine.StringField()
     check_period = mongoengine.StringField()

surveil/api/storage/mongodb/config/command.py

@@ -20,6 +20,6 @@ class Command(mongoengine.Document):
         'collection': 'commands',
         'strict': False
     }
-    command_name = mongoengine.StringField()
+    command_name = mongoengine.StringField(unique=True)
     command_line = mongoengine.StringField()
     module_type = mongoengine.StringField()

surveil/api/storage/mongodb/config/contact.py

@@ -17,7 +17,7 @@ import mongoengine
 
 class Contact(mongoengine.Document):
     meta = {'collection': 'contacts'}
-    contact_name = mongoengine.StringField()
+    contact_name = mongoengine.StringField(unique=True)
     host_notifications_enabled = mongoengine.StringField()
     service_notifications_enabled = mongoengine.StringField()
     host_notification_period = mongoengine.StringField()

surveil/api/storage/mongodb/config/contactgroup.py

@@ -14,10 +14,14 @@
 
 import mongoengine
 
+from surveil.api.storage.mongodb.config import contact
+from surveil.api.storage.mongodb import foreign_key_field
+
 
 class ContactGroup(mongoengine.Document):
     meta = {'collection': 'contactgroups'}
-    contactgroup_name = mongoengine.StringField()
+    contactgroup_name = mongoengine.StringField(unique=True)
-    members = mongoengine.ListField()
+    members = foreign_key_field.ForeignKeyListField(contact.Contact,
+                                                    ['contact_name'])
     alias = mongoengine.StringField()
     contactgroup_members = mongoengine.ListField()

surveil/api/storage/mongodb/config/hostgroup.py

@@ -14,11 +14,15 @@
 
 import mongoengine
 
+from surveil.api.storage.mongodb.config import host
+from surveil.api.storage.mongodb import foreign_key_field
+
 
 class HostGroup(mongoengine.Document):
     meta = {'collection': 'hostgroups'}
     hostgroup_name = mongoengine.StringField(unique=True)
-    members = mongoengine.ListField()
+    members = foreign_key_field.ForeignKeyListField(host.Host,
+                                                    ['host_name', 'name'])
     alias = mongoengine.StringField()
     hostgroup_members = mongoengine.ListField()
     notes = mongoengine.StringField()

surveil/api/storage/mongodb/config/macromodulation.py

@@ -17,6 +17,6 @@ import mongoengine
 
 class MacroModulation(mongoengine.Document):
     meta = {'collection': 'macromodulations'}
-    macromodulation_name = mongoengine.StringField()
+    macromodulation_name = mongoengine.StringField(unique=True)
     modulation_period = mongoengine.StringField()
     macros = mongoengine.DictField()

surveil/api/storage/mongodb/config/notificationway.py

@@ -17,7 +17,7 @@ import mongoengine
 
 class NotificationWays(mongoengine.Document):
     meta = {'collection': 'notificationways'}
-    notificationway_name = mongoengine.StringField()
+    notificationway_name = mongoengine.StringField(unique=True)
     host_notification_period = mongoengine.StringField()
     service_notification_period = mongoengine.StringField()
     host_notification_options = mongoengine.ListField()

surveil/api/storage/mongodb/config/service.py

@@ -14,13 +14,17 @@
 
 import mongoengine
 
+from surveil.api.storage.mongodb.config import host
+from surveil.api.storage.mongodb import foreign_key_field
+
 
 class Service(mongoengine.Document):
     meta = {
         'collection': 'services',
         'strict': False
     }
-    host_name = mongoengine.ListField()
+    host_name = foreign_key_field.ForeignKeyListField(host.Host,
+                                                      ['host_name', 'name'])
     service_description = mongoengine.StringField()
     contacts = mongoengine.ListField()
     check_command = mongoengine.StringField()

surveil/api/storage/mongodb/config/servicegroup.py

@@ -14,11 +14,17 @@
 
 import mongoengine
 
+from surveil.api.storage.mongodb.config import service
+from surveil.api.storage.mongodb import foreign_key_field
+
 
 class ServiceGroup(mongoengine.Document):
     meta = {'collection': 'servicegroups'}
     servicegroup_name = mongoengine.StringField(unique=True)
-    members = mongoengine.ListField()
+    members = foreign_key_field.ForeignKeyListField(
+        service.Service,
+        ['service_description', 'name']
+    )
     alias = mongoengine.StringField()
     servicegroup_members = mongoengine.ListField()
     notes = mongoengine.StringField()

surveil/api/storage/mongodb/foreign_key_field.py

@@ -0,0 +1,71 @@
+# Copyright 2015 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mongoengine
+
+
+def validate_refering_object_exists(referring_object, field_name, value):
+    return referring_object.objects(**{field_name: value}).count() > 0
+
+
+class ForeignKeyStringField(mongoengine.StringField):
+
+    def __init__(self, referring_object,
+                 possible_field_names,
+                 *args,
+                 **kwargs):
+        super(ForeignKeyStringField, self).__init__(*args, **kwargs)
+        self.refering_object = referring_object
+        self.possible_field_names = possible_field_names
+
+    def validate(self, value):
+        mongoengine.StringField.validate(self, value)
+
+        for field in self.possible_field_names:
+            if validate_refering_object_exists(self.refering_object,
+                                               field,
+                                               value) is True:
+                return
+        self.error(
+            "Could not find matching %s" % value
+        )
+
+
+class ForeignKeyListField(mongoengine.ListField):
+    def __init__(self, referring_object,
+                 possible_field_names,
+                 *args,
+                 **kwargs):
+        super(ForeignKeyListField, self).__init__(*args, **kwargs)
+        self.refering_object = referring_object
+        self.possible_field_names = possible_field_names
+
+    def validate(self, value):
+        mongoengine.ListField.validate(self, value)
+
+        for key in value:
+            found_matching = False
+
+            for field in self.possible_field_names:
+                if validate_refering_object_exists(
+                        self.refering_object,
+                        field,
+                        key) is True:
+                    found_matching = True
+                    break
+
+            if found_matching is False:
+                self.error(
+                    "Could not find matching %s" % key
+                )

surveil/cmd/pack_upload.py

@@ -58,7 +58,7 @@ def upload_pack(pack_dir, client):
     surveil_config = surveil_from_nagios.load_config(pack_dir)
 
     config_manager = client.config
-    for object_type, objects in surveil_config.items():
+    for object_type, objects in surveil_config:
         object_manager = getattr(config_manager, object_type)
         for object in objects:
             object_manager.create(**object)

surveil/cmd/surveil_from_nagios.py

@@ -15,6 +15,7 @@
 from __future__ import print_function
 
 import argparse
+import collections
 import fnmatch
 import json
 import os
@@ -73,9 +74,9 @@ def load_config(path):
     else:
         cfg_files = [path]
 
-    nagios_config = {}
+    nagios_config = collections.OrderedDict()
 
-    for cfg_file in cfg_files:
+    for cfg_file in sorted(cfg_files):
         # Open the file
         f = open(cfg_file, 'r')
         config_string = f.read()
@@ -93,12 +94,14 @@ def load_config(path):
 
     surveil_config = _transform_config(nagios_config)
 
-    return surveil_config
+    sorted_surveil_config = _sort_config(surveil_config)
+
+    return sorted_surveil_config
 
 
 def _load_nagios_config(config_string):
     """Given a nagios configuration string, returns a python dict"""
-    config = {}
+    config = collections.OrderedDict()
 
     # Find all config objects
     config_objects = re.finditer(
@@ -132,7 +135,7 @@ def _load_nagios_config(config_string):
 
 def _transform_config(nagios_config):
     """Given a nagios config dict, returns surveil configuration"""
-    transformed_config = {}
+    transformed_config = collections.OrderedDict()
 
     for object_type, objects in nagios_config.items():
         for config_object in objects:
@@ -158,7 +161,7 @@ def _transform_config(nagios_config):
 
 
 def _transform_property_types(config_object, object_type):
-    transformed_object = {}
+    transformed_object = collections.OrderedDict()
 
     datamodels = {
         "businessimpactmodulations": businessimpactmodulation.BusinessImpactModulation,
@@ -194,7 +197,7 @@ def _transform_property_types(config_object, object_type):
 
 
 def _transform_property_names(config_object, object_type):
-    transformed_object = {}
+    transformed_object = collections.OrderedDict()
 
     # HOSTS
     if object_type in ['hosts', 'services']:
@@ -220,7 +223,61 @@ def _transform_property_names(config_object, object_type):
     return transformed_object
 
 
+def _sort_config(surveil_config):
+
+    # Sort object types
+    correct_order = {
+        "realms": 0,
+        "timeperiods": 1,
+        "macromodulations": 2,
+        "commands": 3,
+        "checkmodulations": 4,
+        "businessimpactmodulations": 5,
+        "notificationways": 6,
+        "contacts": 7,
+        "contactgroups": 8,
+        "hosts": 9,
+        "services": 10,
+        "hostgroups": 11,
+        "servicegroups": 12,
+    }
+    sorted_object_types = sorted(surveil_config.items(),
+                                 key=lambda x: correct_order.get(x[0], 99))
+
+    sorted_config = []
+
+    # Sort objects
+    for item in sorted_object_types:
+        object_type = item[0]
+        objects = item[1]
+
+        if object_type in ['hosts', 'services']:
+            objects = _sort_objects(objects)
+
+        sorted_config.append((object_type, objects))
+
+    return sorted_config
+
+
+def _sort_objects(objects):
+    sorted_objects = []
+    while len(objects) > 0:
+        for object in objects:
+            host_dependencies = object.get('use', [])
+            unsolved_dependencies = [
+                d for d in host_dependencies
+                if not any(o.get('name', None) == d
+                           for o in sorted_objects)
+            ]
+
+            if len(unsolved_dependencies) == 0:
+                break
+
+        sorted_objects.append(object)
+        objects.remove(object)
+
+    return sorted_objects
+
+
 if __name__ == "__main__":
     main()
-
-

surveil/tests/api/controllers/v2/config/test_contactgroup.py

@@ -27,12 +27,12 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
         self.groups = [
             {
                 'contactgroup_name': 'novell-admins',
-                'members': ["jdoe", "rtobert", "tzach"],
+                'members': ["bob", "alice"],
                 'contactgroup_members': []
             },
             {
                 'contactgroup_name': 'linux-adminx',
-                'members': ['linus', 'richard'],
+                'members': ['bob'],
                 'contactgroup_members': []
             },
         ]
@@ -40,6 +40,14 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
             copy.deepcopy(self.groups)
         )
 
+        self.contacts = [
+            {"contact_name": "bob"},
+            {"contact_name": "alice"},
+        ]
+        self.mongoconnection.shinken.contacts.insert(
+            copy.deepcopy(self.contacts)
+        )
+
     def test_get_all_contactgroups(self):
         response = self.get('/v2/config/contactgroups')
 
@@ -60,7 +68,7 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
     def test_create_contactgroup(self):
         g = contactgroup.ContactGroup(
             contactgroup_name='John',
-            members=["marie", "bob", "joe"],
+            members=["bob", "alice"],
         )
 
         self.post_json('/v2/config/contactgroups', g.as_dict())
@@ -85,18 +93,18 @@ class TestContactGroupsController(functionalTest.FunctionalTest):
             self.mongoconnection.shinken.contactgroups.find_one(
                 {'contactgroup_name': 'novell-admins'}
             )['members'],
-            ['jdoe', 'rtobert', 'tzach']
+            ['bob', 'alice']
         )
 
         self.put_json(
             '/v2/config/contactgroups/novell-admins',
             {"contactgroup_name": "novell-admins",
-             "members": ["updated"]}
+             "members": ["bob"]}
         )
 
         self.assertEqual(
             self.mongoconnection.shinken.contactgroups.find_one(
                 {'contactgroup_name': 'novell-admins'}
             )['members'],
-            ['updated']
+            ['bob']
         )

surveil/tests/api/controllers/v2/config/test_hostgroup.py

@@ -27,12 +27,12 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
         self.groups = [
             {
                 'hostgroup_name': 'novell-servers',
-                'members': ['netware1', 'netware2', 'netware3', 'netware4'],
+                'members': ['host1'],
                 'hostgroup_members': []
             },
             {
                 'hostgroup_name': 'otherservers',
-                'members': ['googul', 'sfl'],
+                'members': ['host1', 'host2'],
                 'hostgroup_members': []
             },
         ]
@@ -40,6 +40,14 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
             copy.deepcopy(self.groups)
         )
 
+        self.hosts = [
+            {"host_name": 'host1'},
+            {"host_name": 'host2'}
+        ]
+        self.mongoconnection.shinken.hosts.insert(
+            copy.deepcopy(self.hosts)
+        )
+
     def test_get_all_hostgroups(self):
         response = self.get('/v2/config/hostgroups')
 
@@ -60,7 +68,7 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
     def test_create_hostgroup(self):
         s = hostgroup.HostGroup(
             hostgroup_name='John',
-            members=['marie', 'bob', 'joe'],
+            members=['host1', 'host2'],
         )
 
         self.post_json('/v2/config/hostgroups', s.as_dict())
@@ -85,18 +93,18 @@ class TestHostGroupsController(functionalTest.FunctionalTest):
             self.mongoconnection.shinken.hostgroups.find_one(
                 {'hostgroup_name': 'novell-servers'}
             )['members'],
-            ['netware1', 'netware2', 'netware3', 'netware4']
+            ['host1']
         )
 
         self.put_json(
             '/v2/config/hostgroups/novell-servers',
             {"hostgroup_name": "novell-servers",
-             "members": ["updated"]}
+             "members": ["host2"]}
         )
 
         self.assertEqual(
             self.mongoconnection.shinken.hostgroups.find_one(
                 {'hostgroup_name': 'novell-servers'}
             )['members'],
-            ['updated']
+            ['host2']
         )

surveil/tests/api/controllers/v2/config/test_servicegroup.py

@@ -27,19 +27,13 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
         self.groups = [
             {
                 'servicegroup_name': 'dbservices',
-                'members': ['ms1',
+                'members': ['service1'],
-                            'SQL Server',
-                            'ms1',
-                            'SQL Serverc Agent',
-                            'ms1',
-                            'SQL DTC'],
                 'servicegroup_members': []
             },
             {
                 'servicegroup_name': 'otherservices',
-                'members': ['some',
+                'members': ['service1',
-                            'other',
+                            'service2'],
-                            'member'],
                 'servicegroup_members': []
             },
         ]
@@ -47,6 +41,14 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
             copy.deepcopy(self.groups)
         )
 
+        self.services = [
+            {'service_description': "service1"},
+            {'service_description': "service2"}
+        ]
+        self.mongoconnection.shinken.services.insert(
+            copy.deepcopy(self.services)
+        )
+
     def test_get_all_servicegroups(self):
         response = self.get('/v2/config/servicegroups')
 
@@ -67,7 +69,7 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
     def test_create_servicegroup(self):
         s = servicegroup.ServiceGroup(
             servicegroup_name='John',
-            members=['marie', 'bob', 'joe'],
+            members=['service1'],
         )
 
         self.post_json('/v2/config/servicegroups', s.as_dict())
@@ -92,23 +94,18 @@ class TestServiceGroupsController(functionalTest.FunctionalTest):
             self.mongoconnection.shinken.servicegroups.find_one(
                 {'servicegroup_name': 'dbservices'}
             )['members'],
-            ['ms1',
+            ['service1']
-             'SQL Server',
-             'ms1',
-             'SQL Serverc Agent',
-             'ms1',
-             'SQL DTC']
         )
 
         self.put_json(
             '/v2/config/servicegroups/dbservices',
             {"servicegroup_name": "dbservices",
-             "members": ["updated"]}
+             "members": ["service2"]}
         )
 
         self.assertEqual(
             self.mongoconnection.shinken.servicegroups.find_one(
                 {'servicegroup_name': 'dbservices'}
             )['members'],
-            ['updated']
+            ['service2']
         )

surveil/tests/api/controllers/v2/config/test_services.py

@@ -71,6 +71,13 @@ class TestServiceController(functionalTest.FunctionalTest):
             copy.deepcopy(self.services)
         )
 
+        self.hosts = [
+            {"host_name": "sample-server1"}
+        ]
+        self.mongoconnection.shinken.hosts.insert(
+            copy.deepcopy(self.services)
+        )
+
     def test_get_all_services(self):
         response = self.get('/v2/config/services')
 
@@ -115,7 +122,7 @@ class TestServiceController(functionalTest.FunctionalTest):
 
     def test_add_service(self):
         new_service = {
-            "host_name": ["SOMEHOSTNAME"],
+            "host_name": ["sample-server1"],
             "service_description": "check-new-thing",
             "check_command": "check-disk!/dev/sdb1",
             "max_check_attempts": 5,

surveil/tests/api/controllers/v2/config/test_timeperiods.py

@@ -79,7 +79,9 @@ class TestTimePeriodsController(functionalTest.FunctionalTest):
         self.post_json('/v2/config/timeperiods', t)
 
         self.assertIsNotNone(
-            self.mongoconnection.shinken.timeperiods.find_one(t)
+            self.mongoconnection.shinken.timeperiods.find_one(
+                {"timeperiod_name": 'someperiod'}
+            )
         )
 
     def test_delete_timeperiod(self):

surveil/tests/cmd/nagios_config/config.cfg

@@ -1,5 +1,5 @@
   define host{
-          use                     generic-host
+          use                     generic-host,non-existing-thing
       contact_groups      admins
           host_name               localhost
           address                 localhost
@@ -7,6 +7,15 @@
  check_interval          324
           }
 
+
+define host{
+  name generic-host
+}
+
+define host{
+  use generic-host
+}
+
  define service {
      host_name test
  }

surveil/tests/cmd/test_surveil_from_nagios.py

@@ -33,47 +33,47 @@ class TestSurveilFromNagios(base_test.BaseTestCase):
 
         self.assert_count_equal_backport(
             surveil_cfg,
-            {
+            [
-                'hosts': [
+                ('timeperiods',
-                    {
+                 [
-                        'custom_fields': {'_custom_yolo': 'sdfsdf'},
+                     {
-                        'contact_groups': [
+                         'alias': 'Normal Work Hours',
-                            'admins'
+                         'timeperiod_name': 'workhours',
-                        ],
+                         'periods': {
-                        'use': [
+                             'tuesday': '09:00-17:00',
-                            'generic-host'
+                             'friday': '09:00-17:00',
-                        ],
+                             'thursday': '09:00-17:00',
-                        'host_name': 'localhost',
+                             'wednesday': '09:00-17:00',
-                        'check_interval': 324,
+                             'monday': '09:00-17:00'
-                        'address': 'localhost'}
+                         }
-                ],
+                     }
-                'services': [
+                 ]),
-                    {
+                ('hosts',
-                        'host_name': [
+                 [
-                            'hai'
+                     {
-                        ]
+                         'name': 'generic-host',
-                    },
+                         'custom_fields': {}},
-                    {
+                     {
-                        'host_name': [
+                         'use': ['generic-host'],
-                            'test'
+                         'custom_fields': {}},
-                        ]
+                     {
-                    }
+                         'use': ['generic-host', 'non-existing-thing'],
-                ],
+                         'contact_groups': ['admins'],
-                'timeperiods': [
+                         'host_name': 'localhost',
-                    {
+                         'check_interval': 324,
-                        'timeperiod_name': 'workhours',
+                         'address': 'localhost',
-                        'alias': 'Normal Work Hours',
+                         'custom_fields': {
-                        'periods':
+                             '_custom_yolo': 'sdfsdf'
-                            {
+                         }
-                                'friday': '09:00-17:00',
+                     },
-                                'monday': '09:00-17:00',
+
-                                'thursday': '09:00-17:00',
+                 ]),
-                                'tuesday': '09:00-17:00',
+                ('services',
-                                'wednesday': '09:00-17:00'
+                 [
-                            }
+                     {'host_name': ['test']},
-                    }
+                     {'host_name': ['hai']}
-                ]
+                 ])
-            }
+            ]
         )
 
     def test_load_single_file(self):
@@ -88,5 +88,5 @@ class TestSurveilFromNagios(base_test.BaseTestCase):
 
         self.assertEqual(
             single_file_config,
-            {'services': [{'host_name': ['hai']}]}
+            [('services', [{'host_name': ['hai']}])]
         )