Adding Search endpoints and sqlalchemy impl

Search endpoints added for Projects, Stories, Tasks, Comments and Users.

SqlAlchemy plugin for fulltext search added.
The migration for full-text indexes added.
The migration checks the MySQL server version before creating indexes.

The default search engine via sqlalchemy added.

Change-Id: Ie3e4c4f317338d68e82c9c21652d49220c6e4a7d

requirements.txt

@@ -14,4 +14,5 @@ six>=1.7.0
 SQLAlchemy>=0.8,<=0.8.99
 WSME>=0.6
 sqlalchemy-migrate>=0.8.2,!=0.8.4
+SQLAlchemy-FullText-Search
 eventlet>=0.13.0

storyboard/api/app.py

@@ -24,6 +24,8 @@ from storyboard.api.auth.token_storage import storage
 from storyboard.api import config as api_config
 from storyboard.api.middleware import token_middleware
 from storyboard.api.middleware import user_id_hook
+from storyboard.api.v1.search import impls as search_engine_impls
+from storyboard.api.v1.search import search_engine
 from storyboard.openstack.common.gettextutils import _  # noqa
 from storyboard.openstack.common import log
 
@@ -65,6 +67,11 @@ def setup_app(pecan_config=None):
     storage_cls = storage_impls.STORAGE_IMPLS[token_storage_type]
     storage.set_storage(storage_cls())
 
+    # Setup search engine
+    search_engine_name = CONF.search_engine
+    search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name]
+    search_engine.set_engine(search_engine_cls())
+
     app = pecan.make_app(
         pecan_config.app.root,
         debug=CONF.debug,

storyboard/api/v1/projects.py

@@ -24,11 +24,14 @@ import wsmeext.pecan as wsme_pecan
 
 from storyboard.api.auth import authorization_checks as checks
 from storyboard.api.v1 import base
+from storyboard.api.v1.search import search_engine
 from storyboard.common.custom_types import NameType
 from storyboard.db.api import projects as projects_api
 
 CONF = cfg.CONF
 
+SEARCH_ENGINE = search_engine.get_engine()
+
 
 class Project(base.APIBase):
     """The Storyboard Registry describes the open source world as ProjectGroups
@@ -66,6 +69,8 @@ class ProjectsController(rest.RestController):
     At this moment it provides read-only operations.
     """
 
+    _custom_actions = {"search": ["GET"]}
+
     @secure(checks.guest)
     @wsme_pecan.wsexpose(Project, int)
     def get_one_by_id(self, project_id):
@@ -171,12 +176,31 @@ class ProjectsController(rest.RestController):
         except ValueError:
             return False
 
+    @secure(checks.guest)
+    @wsme_pecan.wsexpose([Project], unicode, unicode, int, int)
+    def search(self, q="", marker=None, limit=None):
+        """The search endpoint for projects.
+
+        :param q: The query string.
+        :return: List of Projects matching the query.
+        """
+
+        projects = SEARCH_ENGINE.projects_query(q=q, marker=marker,
+                                                limit=limit)
+
+        return [Project.from_db_model(project) for project in projects]
+
     @expose()
     def _route(self, args, request):
         if request.method == 'GET' and len(args) > 0:
             # It's a request by a name or id
-            first_token = args[0]
+            something = args[0]
-            if self._is_int(first_token):
+
+            if something == "search":
+                # Request to a search endpoint
+                return super(ProjectsController, self)._route(args, request)
+
+            if self._is_int(something):
                 # Get by id
                 return self.get_one_by_id, args
             else:

storyboard/api/v1/search/impls.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from storyboard.api.v1.search.sqlalchemy_impl import SqlAlchemySearchImpl
+
+
+ENGINE_IMPLS = {
+    "sqlalchemy": SqlAlchemySearchImpl
+}

storyboard/api/v1/search/search_engine.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+
+from oslo.config import cfg
+
+from storyboard.db import models
+
+CONF = cfg.CONF
+
+SEARCH_OPTS = [
+    cfg.StrOpt('search_engine',
+               default='sqlalchemy',
+               help='Search engine implementation.'
+                    ' The only supported type is "sqlalchemy".')
+]
+
+CONF.register_opts(SEARCH_OPTS)
+
+
+class SearchEngine(object):
+    """This is an interface that should be implemented by search engines.
+
+    """
+
+    searchable_fields = {
+        models.Project: ["name", "description"],
+        models.Story: ["title", "description"],
+        models.Task: ["title"],
+        models.Comment: ["content"],
+        models.User: ['username', 'full_name', 'email']
+    }
+
+    @abc.abstractmethod
+    def projects_query(self, q, sort_dir=None, marker=None, limit=None,
+                       **kwargs):
+        pass
+
+    @abc.abstractmethod
+    def stories_query(self, q, status=None, author=None,
+                      created_after=None, created_before=None,
+                      updated_after=None, updated_before=None,
+                      marker=None, limit=None, **kwargs):
+        pass
+
+    @abc.abstractmethod
+    def tasks_query(self, q, status=None, author=None, priority=None,
+                    assignee=None, project=None, project_group=None,
+                    created_after=None, created_before=None,
+                    updated_after=None, updated_before=None,
+                    marker=None, limit=None, **kwargs):
+        pass
+
+    @abc.abstractmethod
+    def comments_query(self, q, created_after=None, created_before=None,
+                       updated_after=None, updated_before=None,
+                       marker=None, limit=None, **kwargs):
+        pass
+
+    @abc.abstractmethod
+    def users_query(self, q, marker=None, limit=None, **kwargs):
+        pass
+
+
+ENGINE = None
+
+
+def get_engine():
+    global ENGINE
+    return ENGINE
+
+
+def set_engine(impl):
+    global ENGINE
+    ENGINE = impl

storyboard/api/v1/search/sqlalchemy_impl.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo.db.sqlalchemy import utils
+from sqlalchemy_fulltext import FullTextSearch
+import sqlalchemy_fulltext.modes as FullTextMode
+
+from storyboard.api.v1.search import search_engine
+from storyboard.db.api import base as api_base
+from storyboard.db import models
+
+
+class SqlAlchemySearchImpl(search_engine.SearchEngine):
+
+    def _build_fulltext_search(self, model_cls, query, q):
+        query = query.filter(FullTextSearch(q, model_cls,
+                                            mode=FullTextMode.NATURAL))
+
+        return query
+
+    def _apply_pagination(self, model_cls, query, marker=None, limit=None):
+
+        marker_entity = None
+        if marker:
+            marker_entity = api_base.entity_get(model_cls, marker, True)
+
+        return utils.paginate_query(query=query,
+                                    model=model_cls,
+                                    limit=limit,
+                                    sort_keys=["id"],
+                                    marker=marker_entity)
+
+    def projects_query(self, q, sort_dir=None, marker=None, limit=None):
+        session = api_base.get_session()
+        query = api_base.model_query(models.Project, session)
+        query = self._build_fulltext_search(models.Project, query, q)
+        query = self._apply_pagination(models.Project, query, marker, limit)
+
+        return query.all()
+
+    def stories_query(self, q, marker=None, limit=None, **kwargs):
+        session = api_base.get_session()
+        query = api_base.model_query(models.Story, session)
+        query = self._build_fulltext_search(models.Story, query, q)
+        query = self._apply_pagination(models.Story, query, marker, limit)
+
+        return query.all()
+
+    def tasks_query(self, q, marker=None, limit=None, **kwargs):
+        session = api_base.get_session()
+        query = api_base.model_query(models.Task, session)
+        query = self._build_fulltext_search(models.Task, query, q)
+        query = self._apply_pagination(models.Task, query, marker, limit)
+
+        return query.all()
+
+    def comments_query(self, q, marker=None, limit=None, **kwargs):
+        session = api_base.get_session()
+        query = api_base.model_query(models.Comment, session)
+        query = self._build_fulltext_search(models.Comment, query, q)
+        query = self._apply_pagination(models.Comment, query, marker, limit)
+
+        return query.all()
+
+    def users_query(self, q, marker=None, limit=None, **kwargs):
+        session = api_base.get_session()
+        query = api_base.model_query(models.User, session)
+        query = self._build_fulltext_search(models.User, query, q)
+        query = self._apply_pagination(models.User, query, marker, limit)
+
+        return query.all()

storyboard/api/v1/stories.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from oslo.config import cfg
+from pecan import expose
 from pecan import request
 from pecan import response
 from pecan import rest
@@ -24,6 +25,7 @@ import wsmeext.pecan as wsme_pecan
 
 from storyboard.api.auth import authorization_checks as checks
 from storyboard.api.v1 import base
+from storyboard.api.v1.search import search_engine
 from storyboard.api.v1.timeline import CommentsController
 from storyboard.api.v1.timeline import TimeLineEventsController
 from storyboard.db.api import stories as stories_api
@@ -31,6 +33,8 @@ from storyboard.db.api import timeline_events as events_api
 
 CONF = cfg.CONF
 
+SEARCH_ENGINE = search_engine.get_engine()
+
 
 class Story(base.APIBase):
     """The Story is the main element of StoryBoard. It represents a user story
@@ -87,6 +91,8 @@ class Story(base.APIBase):
 class StoriesController(rest.RestController):
     """Manages operations on stories."""
 
+    _custom_actions = {"search": ["GET"]}
+
     @secure(checks.guest)
     @wsme_pecan.wsexpose(Story, int)
     def get_one(self, story_id):
@@ -214,3 +220,30 @@ class StoriesController(rest.RestController):
 
     comments = CommentsController()
     events = TimeLineEventsController()
+
+    @secure(checks.guest)
+    @wsme_pecan.wsexpose([Story], unicode, unicode, int, int)
+    def search(self, q="", marker=None, limit=None):
+        """The search endpoint for stories.
+
+        :param q: The query string.
+        :return: List of Stories matching the query.
+        """
+
+        stories = SEARCH_ENGINE.stories_query(q=q,
+                                              marker=marker,
+                                              limit=limit)
+
+        return [Story.from_db_model(story) for story in stories]
+
+    @expose()
+    def _route(self, args, request):
+        if request.method == 'GET' and len(args) > 0:
+            # It's a request by a name or id
+            something = args[0]
+
+            if something == "search":
+                # Request to a search endpoint
+                return self.search, args
+
+        return super(StoriesController, self)._route(args, request)

storyboard/api/v1/tasks.py

@@ -24,11 +24,14 @@ import wsmeext.pecan as wsme_pecan
 
 from storyboard.api.auth import authorization_checks as checks
 from storyboard.api.v1 import base
+from storyboard.api.v1.search import search_engine
 from storyboard.db.api import tasks as tasks_api
 from storyboard.db.api import timeline_events as events_api
 
 CONF = cfg.CONF
 
+SEARCH_ENGINE = search_engine.get_engine()
+
 
 class Task(base.APIBase):
     """A Task represents an actionable work item, targeting a specific Project
@@ -69,6 +72,8 @@ class Task(base.APIBase):
 class TasksController(rest.RestController):
     """Manages tasks."""
 
+    _custom_actions = {"search": ["GET"]}
+
     @secure(checks.guest)
     @wsme_pecan.wsexpose(Task, int)
     def get_one(self, task_id):
@@ -222,3 +227,18 @@ class TasksController(rest.RestController):
         tasks_api.task_delete(task_id)
 
         response.status_code = 204
+
+    @secure(checks.guest)
+    @wsme_pecan.wsexpose([Task], unicode, unicode, int, int)
+    def search(self, q="", marker=None, limit=None):
+        """The search endpoint for tasks.
+
+        :param q: The query string.
+        :return: List of Tasks matching the query.
+        """
+
+        tasks = SEARCH_ENGINE.tasks_query(q=q,
+                                          marker=marker,
+                                          limit=limit)
+
+        return [Task.from_db_model(task) for task in tasks]

storyboard/api/v1/timeline.py

@@ -24,6 +24,7 @@ import wsmeext.pecan as wsme_pecan
 
 from storyboard.api.auth import authorization_checks as checks
 from storyboard.api.v1 import base
+from storyboard.api.v1.search import search_engine
 from storyboard.common import event_resolvers
 from storyboard.common import event_types
 from storyboard.db.api import comments as comments_api
@@ -31,6 +32,8 @@ from storyboard.db.api import timeline_events as events_api
 
 CONF = cfg.CONF
 
+SEARCH_ENGINE = search_engine.get_engine()
+
 
 class Comment(base.APIBase):
     """Any user may leave comments for stories. Also comments api is used by
@@ -300,3 +303,18 @@ class CommentsController(rest.RestController):
 
         response.status_code = 204
         return response
+
+    @secure(checks.guest)
+    @wsme_pecan.wsexpose([Comment], unicode, unicode, int, int)
+    def search(self, q="", marker=None, limit=None):
+        """The search endpoint for comments.
+
+        :param q: The query string.
+        :return: List of Comments matching the query.
+        """
+
+        comments = SEARCH_ENGINE.comments_query(q=q,
+                                                marker=marker,
+                                                limit=limit)
+
+        return [Comment.from_db_model(comment) for comment in comments]

storyboard/api/v1/users.py

@@ -16,6 +16,7 @@
 from datetime import datetime
 
 from oslo.config import cfg
+from pecan import expose
 from pecan import request
 from pecan import response
 from pecan import rest
@@ -26,10 +27,13 @@ import wsmeext.pecan as wsme_pecan
 
 from storyboard.api.auth import authorization_checks as checks
 from storyboard.api.v1 import base
+from storyboard.api.v1.search import search_engine
 from storyboard.db.api import users as users_api
 
 CONF = cfg.CONF
 
+SEARCH_ENGINE = search_engine.get_engine()
+
 
 class User(base.APIBase):
     """Represents a user."""
@@ -69,6 +73,8 @@ class User(base.APIBase):
 class UsersController(rest.RestController):
     """Manages users."""
 
+    _custom_actions = {"search": ["GET"]}
+
     @secure(checks.guest)
     @wsme_pecan.wsexpose([User], int, int, unicode, unicode, unicode, unicode)
     def get(self, marker=None, limit=None, username=None, full_name=None,
@@ -158,3 +164,30 @@ class UsersController(rest.RestController):
 
         updated_user = users_api.user_update(user_id, user_dict)
         return User.from_db_model(updated_user)
+
+    @secure(checks.guest)
+    @wsme_pecan.wsexpose([User], unicode, unicode, int, int)
+    def search(self, q="", marker=None, limit=None):
+        """The search endpoint for users.
+
+        :param q: The query string.
+        :return: List of Users matching the query.
+        """
+
+        users = SEARCH_ENGINE.users_query(q=q, marker=marker, limit=limit)
+
+        return [User.from_db_model(u) for u in users]
+
+    @expose()
+    def _route(self, args, request):
+        if request.method == 'GET' and len(args) > 0:
+            # It's a request by a name or id
+            something = args[0]
+
+            if something == "search":
+                # Request to a search endpoint
+                return self.search, args
+            else:
+                return self.get_one, args
+
+        return super(UsersController, self)._route(args, request)

storyboard/db/migration/alembic_migrations/versions/022_fulltext_indexes.py

@@ -0,0 +1,71 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+"""Adding full text indexes
+
+Revision ID: 022
+Revises: 021
+Create Date: 2014-07-11 14:08:08.129484
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '022'
+down_revision = '021'
+
+
+from alembic import op
+
+from storyboard.openstack.common import log
+
+LOG = log.getLogger(__name__)
+
+
+def upgrade(active_plugins=None, options=None):
+
+    version_info = op.get_bind().engine.dialect.server_version_info
+    if version_info[0] < 5 or version_info[0] == 5 and version_info[1] < 6:
+        LOG.warn("MySQL version is lower than 5.6. Skipping full-text indexes")
+        return
+
+    # Index for projects
+    op.execute("ALTER TABLE projects "
+               "ADD FULLTEXT projects_fti (name, description)")
+
+    # Index for stories
+    op.execute("ALTER TABLE stories "
+               "ADD FULLTEXT stories_fti (title, description)")
+
+    # Index for tasks
+    op.execute("ALTER TABLE tasks ADD FULLTEXT tasks_fti (title)")
+
+    # Index for comments
+    op.execute("ALTER TABLE comments ADD FULLTEXT comments_fti (content)")
+
+    # Index for users
+    op.execute("ALTER TABLE users "
+               "ADD FULLTEXT users_fti (username, full_name, email)")
+
+
+def downgrade(active_plugins=None, options=None):
+
+    version_info = op.get_bind().engine.dialect.server_version_info
+    if version_info[0] < 5 or version_info[0] == 5 and version_info[1] < 6:
+        LOG.warn("MySQL version is lower than 5.6. Skipping full-text indexes")
+        return
+
+    op.drop_index("projects_fti", table_name='projects')
+    op.drop_index("stories_fti", table_name='stories')
+    op.drop_index("tasks_fti", table_name='tasks')
+    op.drop_index("comments_fti", table_name='comments')
+    op.drop_index("users_fti", table_name='users')

storyboard/db/models.py

@@ -36,6 +36,7 @@ from sqlalchemy import String
 from sqlalchemy import Table
 from sqlalchemy import Unicode
 from sqlalchemy import UnicodeText
+from sqlalchemy_fulltext import FullText
 
 
 CONF = cfg.CONF
@@ -91,11 +92,14 @@ team_membership = Table(
 )
 
 
-class User(Base):
+class User(FullText, Base):
     __table_args__ = (
         schema.UniqueConstraint('username', name='uniq_user_username'),
         schema.UniqueConstraint('email', name='uniq_user_email'),
     )
+
+    __fulltext_columns__ = ['username', 'full_name', 'email']
+
     username = Column(Unicode(30))
     full_name = Column(Unicode(255), nullable=True)
     email = Column(String(255))
@@ -134,13 +138,15 @@ class Permission(Base):
 
 
 # TODO(mordred): Do we really need name and title?
-class Project(Base):
+class Project(FullText, Base):
     """Represents a software project."""
 
     __table_args__ = (
         schema.UniqueConstraint('name', name='uniq_project_name'),
     )
 
+    __fulltext_columns__ = ['name', 'description']
+
     name = Column(String(50))
     description = Column(UnicodeText())
     team_id = Column(Integer, ForeignKey('teams.id'))
@@ -164,9 +170,11 @@ class ProjectGroup(Base):
     _public_fields = ["id", "name", "title", "projects"]
 
 
-class Story(Base):
+class Story(FullText, Base):
     __tablename__ = 'stories'
 
+    __fulltext_columns__ = ['title', 'description']
+
     creator_id = Column(Integer, ForeignKey('users.id'))
     creator = relationship(User, primaryjoin=creator_id == User.id)
     title = Column(Unicode(100))
@@ -180,7 +188,9 @@ class Story(Base):
                       "tasks", "events", "tags"]
 
 
-class Task(Base):
+class Task(FullText, Base):
+    __fulltext_columns__ = ['title']
+
     _TASK_STATUSES = ('todo', 'inprogress', 'invalid', 'review', 'merged')
     _TASK_PRIORITIES = ('low', 'medium', 'high')
 
@@ -285,7 +295,8 @@ class TimeLineEvent(Base):
     event_info = Column(UnicodeText(), nullable=True)
 
 
-class Comment(Base):
+class Comment(FullText, Base):
+    __fulltext_columns__ = ['content']
 
     content = Column(UnicodeText)
     is_active = Column(Boolean, default=True)