Merge "Implemented search"

surveil/api/datamodel/live_query.py

@@ -37,6 +37,8 @@ class LiveQuery(types.Base):
     paging = wsme.wsattr(paging.Paging, mandatory=False)
     "Paging."
 
+    search = wsme.wsattr(wtypes.text, mandatory=False)
+
     @classmethod
     def sample(cls):
         return cls(
@@ -60,5 +62,6 @@ class LiveQuery(types.Base):
                 "defined": {
                     "name": True
                 }
-            })
+            }),
+            search='web'
         )

surveil/api/handlers/config/mongoengine_query.py

@@ -14,34 +14,72 @@
 
 import json
 
+import mongoengine
 
-def build_mongoengine_query(live_query):
+from surveil.api.storage.mongodb import foreign_key_field
 
-    #  Build the filters
+
-    query = {}
+def build_mongoengine_query(live_query, resource_storage):
-    kwargs = None
+
+    query = mongoengine.Q()
+
+    # Filters
+    if live_query.filters and json.loads(live_query.filters).items():
+        for filter_name, filter_data in json.loads(live_query.filters).items():
+            for field, value in filter_data.items():
+                qobj = mongoengine.Q(
+                    **_get_mongoengine_filter(field,
+                                              filter_name,
+                                              value)
+                )
+                query = query & qobj
+
+    # search
+    if live_query.search:
+        search_q = None
+
+        string_fields = [
+            field for field in resource_storage._fields
+            if isinstance(
+                getattr(resource_storage, field),
+                (
+                    mongoengine.StringField,
+                    mongoengine.ListField,
+                    foreign_key_field.ForeignKeyListField,
+                    foreign_key_field.ForeignKeyListField
+                )
+            )
+        ]
+
+        for field in string_fields:
+            field_q = mongoengine.Q(
+                __raw__={
+                    field: {"$regex": ".*%s.*" % live_query.search,
+                            "$options": "-i"}
+                }
+            )
+
+            if search_q is None:
+                search_q = field_q
+            else:
+                search_q = search_q | field_q
+
+        query = query & search_q
+
+    # Fields
     fields = []
-
     if live_query.fields:
         for field in live_query.fields:
             fields.append(field)
 
-    if live_query.filters and json.loads(live_query.filters).items():
+    # Paging
-        for filter_name, filter_data in json.loads(live_query.filters).items():
+    skip = None
-            for field, value in filter_data.items():
+    limit = None
-                query.update(_get_mongoengine_filter(field,
-                                                     filter_name,
-                                                     value))
-
-    live_query.paging
     if live_query.paging:
-        paging = live_query.paging
+        skip = live_query.paging.size * live_query.paging.page
-        skip = paging.size * paging.page
+        limit = skip + live_query.paging.size
-        limit = skip + paging.size
+
-        kwargs = slice(skip, limit)
+    return fields, query, skip, limit
-    else:
-        kwargs = slice(None, None)
-    return fields, query, kwargs
 
 
 def _get_mongoengine_filter(field_name, filter_name, value):

surveil/api/handlers/mongodb_mongoengine_object_handler.py

@@ -64,24 +64,36 @@ class MongoObjectHandler(handler.Handler):
     def get_all(self, lq={}):
         """Return all resources."""
 
-        fields, query, kwargs = mongoengine_query.build_mongoengine_query(lq)
+        fields, query, skip, limit = mongoengine_query.build_mongoengine_query(
+            lq, self.resource_storage
+        )
+
+        if skip is not None and limit is not None:
+            objects = (
+                self.resource_storage.objects
+                .filter(query)
+                .only(*fields)
+                .skip(skip)
+                .limit(limit)
+            )
+        else:
+            objects = self.resource_storage.objects.filter(query).only(*fields)
 
         resp = [
             self.resource_datamodel(**self._get_dict(r))
             for r
-            in self.resource_storage.objects(**query)
+            in objects
-        ][kwargs]
+        ]
-
-        resp_field = []
 
+        # Mongoengine's 'only()' does not seem to work :(
+        filtered_response = []
         if fields:
             for obj in resp:
                 obj_with_field = {}
                 for field in fields:
-                    obj_with_field[field] = obj[field]
+                    obj_with_field[field] = getattr(obj, field)
-                resp_field.append(obj_with_field)
+                filtered_response.append(
+                    self.resource_datamodel(**obj_with_field)
+                )
 
-        else:
+        return filtered_response or resp
-            resp_field = resp
-
-        return resp_field

surveil/api/handlers/status/live_host_handler.py

@@ -13,11 +13,11 @@
 # under the License.
 
 from surveil.api.datamodel.status import live_host
-from surveil.api.handlers import handler
 from surveil.api.handlers.status import mongodb_query
+from surveil.api.handlers.status import status_handler
 
 
-class HostHandler(handler.Handler):
+class HostHandler(status_handler.StatusHandler):
     """Fulfills a request on the live hosts."""
 
     def get(self, host_name):

surveil/api/handlers/status/live_service_handler.py

@@ -13,11 +13,11 @@
 # under the License.
 
 from surveil.api.datamodel.status import live_service
-from surveil.api.handlers import handler
 from surveil.api.handlers.status import mongodb_query
+from surveil.api.handlers.status import status_handler
 
 
-class ServiceHandler(handler.Handler):
+class ServiceHandler(status_handler.StatusHandler):
     """Fulfills a request on live services."""
 
     def get(self, host_name, service_description):

surveil/api/handlers/status/mongodb_query.py

@@ -27,6 +27,10 @@ def build_mongodb_query(live_query):
                 _get_mongo_filter(filter_name): values
             }
 
+    search = live_query.get('search', None)
+    if search:
+        filters["$text"] = {"$search": search}
+
     if filters:
         query.append(filters)
 

surveil/api/handlers/status/status_handler.py

@@ -0,0 +1,40 @@
+# Copyright 2015 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import pymongo
+
+from surveil.api.handlers import handler
+
+
+class StatusHandler(handler.Handler):
+    """This handler creates MongoDB indexes."""
+
+    def __init__(self, *args, **kwargs):
+        super(StatusHandler, self).__init__(*args, **kwargs)
+
+        try:
+            self.request.mongo_connection.admin.command(
+                'setParameter',
+                textSearchEnabled=True
+            )
+        except Exception:
+            pass
+
+        self.request.mongo_connection.alignak_live.hosts.ensure_index(
+            [("$**", pymongo.TEXT)]
+        )
+
+        self.request.mongo_connection.alignak_live.services.ensure_index(
+            [("$**", pymongo.TEXT)]
+        )

surveil/tests/api/config/test_mongolive_query.py

@@ -17,12 +17,13 @@ import json
 from surveil.api.datamodel import live_query
 from surveil.api.datamodel.status import paging
 from surveil.api.handlers.config import mongoengine_query
+from surveil.api.storage.mongodb.config import host
 from surveil.tests import base
 
 
-class MongoliveQueryTest(base.BaseTestCase):
+class MongoEngineliveQueryTest(base.BaseTestCase):
 
-    def test_build_mongo_query(self):
+    def test_build_mongoengine_query(self):
         query = live_query.LiveQuery(
             fields=['host_name', 'last_check'],
             filters=json.dumps({
@@ -44,23 +45,15 @@ class MongoliveQueryTest(base.BaseTestCase):
 
         )
 
-        fields, query, kwargs = mongoengine_query.build_mongoengine_query(
+        fields, query, skip, limit = mongoengine_query.build_mongoengine_query(
-            query)
+            query,
+            host.Host
+        )
 
         self.assertEqual(
             fields,
             ['host_name', 'last_check']
         )
 
-        self.assertEqual(
+        self.assertEqual(skip, 300)
-            query,
+        self.assertEqual(limit, 400)
-            {"state__nin": ["0", "1"],
-             "host_state__nin": ["2"],
-             "event_type__in": ["ALERT"],
-             "name__exists": True, }
-        )
-
-        self.assertEqual(
-            kwargs,
-            slice(300, 400)
-        )

surveil/tests/api/controllers/v2/config/test_hosts.py

@@ -96,6 +96,19 @@ class TestHostController(functionalTest.FunctionalTest):
         )
         self.assertEqual(response.status_int, 200)
 
+    def test_get_all_hosts_paging(self):
+        response = self.post_json(
+            '/v2/config/hosts',
+            params={"paging": {"page": 2, "size": 1}}
+        )
+
+        hosts = json.loads(response.body.decode())
+
+        self.assertEqual(
+            hosts,
+            [self.hosts[2]]
+        )
+
     def test_get_all_hosts_templates(self):
         self.mongoconnection.shinken.hosts.insert(
             copy.deepcopy(

surveil/tests/api/controllers/v2/config/test_services.py

@@ -35,7 +35,7 @@ class TestServiceController(functionalTest.FunctionalTest):
                 "notification_interval": 30,
                 "notification_period": "24x7",
                 "contacts": ["surveil-ptl", "surveil-bob"],
-                "contact_groups": ["linux-admins"],
+                "contact_groups": ["linux-masters"],
                 "use": []
             },
             {
@@ -87,6 +87,20 @@ class TestServiceController(functionalTest.FunctionalTest):
         )
         self.assertEqual(response.status_int, 200)
 
+    def test_search(self):
+        response = self.post_json('/v2/config/services', params={
+            "search": "admins",
+            "fields": ['service_description']
+        })
+
+        self.assert_count_equal_backport(
+            json.loads(response.body.decode()),
+            [
+                {'service_description': 'check-disk-sdb2'},
+                {'service_description': 'check-disk-sdb3'},
+            ]
+        )
+
     def test_get_all_services_templates(self):
         self.mongoconnection.shinken.services.insert(
             copy.deepcopy(

surveil/tests/api/controllers/v2/status/test_hosts.py

@@ -14,6 +14,7 @@
 
 import copy
 import json
+import unittest
 
 import requests_mock
 from six.moves import urllib_parse
@@ -165,6 +166,20 @@ class TestStatusHosts(functionalTest.FunctionalTest):
         self.assert_count_equal_backport(json.loads(response.body.decode()),
                                          expected)
 
+    @unittest.skip("Does not work on jenkins")
+    def test_search_hosts(self):
+        query = {
+            'fields': ['host_name'],
+            'search': 'another'
+        }
+
+        response = self.post_json("/v2/status/hosts", params=query)
+
+        self.assertEqual(
+            json.loads(response.body.decode()),
+            [{"host_name": "test_keystone"}]
+        )
+
     def test_query_host_paging(self):
         query = {
             'paging': {