From 860f12ff00b05b6c12194e061a62ef8ce2514566 Mon Sep 17 00:00:00 2001 From: Nikita Konovalov Date: Fri, 20 Jun 2014 14:52:36 +0400 Subject: [PATCH] Adding Search endpoints and sqlalchemy impl Search endpoints added for Projects, Stories, Tasks, Comments and Users. SqlAlchemy plugin for fulltext search added. The migration for full-text indexes added. The migration checks the MySQL server version before creating indexes. The default search engine via sqlalchemy added. Change-Id: Ie3e4c4f317338d68e82c9c21652d49220c6e4a7d --- requirements.txt | 1 + storyboard/api/app.py | 7 ++ storyboard/api/v1/projects.py | 28 +++++- storyboard/api/v1/search/__init__.py | 0 storyboard/api/v1/search/impls.py | 21 +++++ storyboard/api/v1/search/search_engine.py | 88 +++++++++++++++++++ storyboard/api/v1/search/sqlalchemy_impl.py | 83 +++++++++++++++++ storyboard/api/v1/stories.py | 33 +++++++ storyboard/api/v1/tasks.py | 20 +++++ storyboard/api/v1/timeline.py | 18 ++++ storyboard/api/v1/users.py | 33 +++++++ .../versions/022_fulltext_indexes.py | 71 +++++++++++++++ storyboard/db/models.py | 21 +++-- 13 files changed, 417 insertions(+), 7 deletions(-) create mode 100644 storyboard/api/v1/search/__init__.py create mode 100644 storyboard/api/v1/search/impls.py create mode 100644 storyboard/api/v1/search/search_engine.py create mode 100644 storyboard/api/v1/search/sqlalchemy_impl.py create mode 100644 storyboard/db/migration/alembic_migrations/versions/022_fulltext_indexes.py diff --git a/requirements.txt b/requirements.txt index 9968fdb7..e5d67e15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,5 @@ six>=1.7.0 SQLAlchemy>=0.8,<=0.8.99 WSME>=0.6 sqlalchemy-migrate>=0.8.2,!=0.8.4 +SQLAlchemy-FullText-Search eventlet>=0.13.0 diff --git a/storyboard/api/app.py b/storyboard/api/app.py index 9df7a9ba..58bfaefa 100644 --- a/storyboard/api/app.py +++ b/storyboard/api/app.py @@ -24,6 +24,8 @@ from storyboard.api.auth.token_storage import storage from storyboard.api import config as api_config from storyboard.api.middleware import token_middleware from storyboard.api.middleware import user_id_hook +from storyboard.api.v1.search import impls as search_engine_impls +from storyboard.api.v1.search import search_engine from storyboard.openstack.common.gettextutils import _ # noqa from storyboard.openstack.common import log @@ -65,6 +67,11 @@ def setup_app(pecan_config=None): storage_cls = storage_impls.STORAGE_IMPLS[token_storage_type] storage.set_storage(storage_cls()) + # Setup search engine + search_engine_name = CONF.search_engine + search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name] + search_engine.set_engine(search_engine_cls()) + app = pecan.make_app( pecan_config.app.root, debug=CONF.debug, diff --git a/storyboard/api/v1/projects.py b/storyboard/api/v1/projects.py index 3ac6ccf9..64a6b00b 100644 --- a/storyboard/api/v1/projects.py +++ b/storyboard/api/v1/projects.py @@ -24,11 +24,14 @@ import wsmeext.pecan as wsme_pecan from storyboard.api.auth import authorization_checks as checks from storyboard.api.v1 import base +from storyboard.api.v1.search import search_engine from storyboard.common.custom_types import NameType from storyboard.db.api import projects as projects_api CONF = cfg.CONF +SEARCH_ENGINE = search_engine.get_engine() + class Project(base.APIBase): """The Storyboard Registry describes the open source world as ProjectGroups @@ -66,6 +69,8 @@ class ProjectsController(rest.RestController): At this moment it provides read-only operations. """ + _custom_actions = {"search": ["GET"]} + @secure(checks.guest) @wsme_pecan.wsexpose(Project, int) def get_one_by_id(self, project_id): @@ -171,12 +176,31 @@ class ProjectsController(rest.RestController): except ValueError: return False + @secure(checks.guest) + @wsme_pecan.wsexpose([Project], unicode, unicode, int, int) + def search(self, q="", marker=None, limit=None): + """The search endpoint for projects. + + :param q: The query string. + :return: List of Projects matching the query. + """ + + projects = SEARCH_ENGINE.projects_query(q=q, marker=marker, + limit=limit) + + return [Project.from_db_model(project) for project in projects] + @expose() def _route(self, args, request): if request.method == 'GET' and len(args) > 0: # It's a request by a name or id - first_token = args[0] - if self._is_int(first_token): + something = args[0] + + if something == "search": + # Request to a search endpoint + return super(ProjectsController, self)._route(args, request) + + if self._is_int(something): # Get by id return self.get_one_by_id, args else: diff --git a/storyboard/api/v1/search/__init__.py b/storyboard/api/v1/search/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/storyboard/api/v1/search/impls.py b/storyboard/api/v1/search/impls.py new file mode 100644 index 00000000..7621c22f --- /dev/null +++ b/storyboard/api/v1/search/impls.py @@ -0,0 +1,21 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from storyboard.api.v1.search.sqlalchemy_impl import SqlAlchemySearchImpl + + +ENGINE_IMPLS = { + "sqlalchemy": SqlAlchemySearchImpl +} diff --git a/storyboard/api/v1/search/search_engine.py b/storyboard/api/v1/search/search_engine.py new file mode 100644 index 00000000..406072ca --- /dev/null +++ b/storyboard/api/v1/search/search_engine.py @@ -0,0 +1,88 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + +from oslo.config import cfg + +from storyboard.db import models + +CONF = cfg.CONF + +SEARCH_OPTS = [ + cfg.StrOpt('search_engine', + default='sqlalchemy', + help='Search engine implementation.' + ' The only supported type is "sqlalchemy".') +] + +CONF.register_opts(SEARCH_OPTS) + + +class SearchEngine(object): + """This is an interface that should be implemented by search engines. + + """ + + searchable_fields = { + models.Project: ["name", "description"], + models.Story: ["title", "description"], + models.Task: ["title"], + models.Comment: ["content"], + models.User: ['username', 'full_name', 'email'] + } + + @abc.abstractmethod + def projects_query(self, q, sort_dir=None, marker=None, limit=None, + **kwargs): + pass + + @abc.abstractmethod + def stories_query(self, q, status=None, author=None, + created_after=None, created_before=None, + updated_after=None, updated_before=None, + marker=None, limit=None, **kwargs): + pass + + @abc.abstractmethod + def tasks_query(self, q, status=None, author=None, priority=None, + assignee=None, project=None, project_group=None, + created_after=None, created_before=None, + updated_after=None, updated_before=None, + marker=None, limit=None, **kwargs): + pass + + @abc.abstractmethod + def comments_query(self, q, created_after=None, created_before=None, + updated_after=None, updated_before=None, + marker=None, limit=None, **kwargs): + pass + + @abc.abstractmethod + def users_query(self, q, marker=None, limit=None, **kwargs): + pass + + +ENGINE = None + + +def get_engine(): + global ENGINE + return ENGINE + + +def set_engine(impl): + global ENGINE + ENGINE = impl diff --git a/storyboard/api/v1/search/sqlalchemy_impl.py b/storyboard/api/v1/search/sqlalchemy_impl.py new file mode 100644 index 00000000..c72bb936 --- /dev/null +++ b/storyboard/api/v1/search/sqlalchemy_impl.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.db.sqlalchemy import utils +from sqlalchemy_fulltext import FullTextSearch +import sqlalchemy_fulltext.modes as FullTextMode + +from storyboard.api.v1.search import search_engine +from storyboard.db.api import base as api_base +from storyboard.db import models + + +class SqlAlchemySearchImpl(search_engine.SearchEngine): + + def _build_fulltext_search(self, model_cls, query, q): + query = query.filter(FullTextSearch(q, model_cls, + mode=FullTextMode.NATURAL)) + + return query + + def _apply_pagination(self, model_cls, query, marker=None, limit=None): + + marker_entity = None + if marker: + marker_entity = api_base.entity_get(model_cls, marker, True) + + return utils.paginate_query(query=query, + model=model_cls, + limit=limit, + sort_keys=["id"], + marker=marker_entity) + + def projects_query(self, q, sort_dir=None, marker=None, limit=None): + session = api_base.get_session() + query = api_base.model_query(models.Project, session) + query = self._build_fulltext_search(models.Project, query, q) + query = self._apply_pagination(models.Project, query, marker, limit) + + return query.all() + + def stories_query(self, q, marker=None, limit=None, **kwargs): + session = api_base.get_session() + query = api_base.model_query(models.Story, session) + query = self._build_fulltext_search(models.Story, query, q) + query = self._apply_pagination(models.Story, query, marker, limit) + + return query.all() + + def tasks_query(self, q, marker=None, limit=None, **kwargs): + session = api_base.get_session() + query = api_base.model_query(models.Task, session) + query = self._build_fulltext_search(models.Task, query, q) + query = self._apply_pagination(models.Task, query, marker, limit) + + return query.all() + + def comments_query(self, q, marker=None, limit=None, **kwargs): + session = api_base.get_session() + query = api_base.model_query(models.Comment, session) + query = self._build_fulltext_search(models.Comment, query, q) + query = self._apply_pagination(models.Comment, query, marker, limit) + + return query.all() + + def users_query(self, q, marker=None, limit=None, **kwargs): + session = api_base.get_session() + query = api_base.model_query(models.User, session) + query = self._build_fulltext_search(models.User, query, q) + query = self._apply_pagination(models.User, query, marker, limit) + + return query.all() diff --git a/storyboard/api/v1/stories.py b/storyboard/api/v1/stories.py index 9476d7b3..3f331bd9 100644 --- a/storyboard/api/v1/stories.py +++ b/storyboard/api/v1/stories.py @@ -14,6 +14,7 @@ # limitations under the License. from oslo.config import cfg +from pecan import expose from pecan import request from pecan import response from pecan import rest @@ -24,6 +25,7 @@ import wsmeext.pecan as wsme_pecan from storyboard.api.auth import authorization_checks as checks from storyboard.api.v1 import base +from storyboard.api.v1.search import search_engine from storyboard.api.v1.timeline import CommentsController from storyboard.api.v1.timeline import TimeLineEventsController from storyboard.db.api import stories as stories_api @@ -31,6 +33,8 @@ from storyboard.db.api import timeline_events as events_api CONF = cfg.CONF +SEARCH_ENGINE = search_engine.get_engine() + class Story(base.APIBase): """The Story is the main element of StoryBoard. It represents a user story @@ -87,6 +91,8 @@ class Story(base.APIBase): class StoriesController(rest.RestController): """Manages operations on stories.""" + _custom_actions = {"search": ["GET"]} + @secure(checks.guest) @wsme_pecan.wsexpose(Story, int) def get_one(self, story_id): @@ -214,3 +220,30 @@ class StoriesController(rest.RestController): comments = CommentsController() events = TimeLineEventsController() + + @secure(checks.guest) + @wsme_pecan.wsexpose([Story], unicode, unicode, int, int) + def search(self, q="", marker=None, limit=None): + """The search endpoint for stories. + + :param q: The query string. + :return: List of Stories matching the query. + """ + + stories = SEARCH_ENGINE.stories_query(q=q, + marker=marker, + limit=limit) + + return [Story.from_db_model(story) for story in stories] + + @expose() + def _route(self, args, request): + if request.method == 'GET' and len(args) > 0: + # It's a request by a name or id + something = args[0] + + if something == "search": + # Request to a search endpoint + return self.search, args + + return super(StoriesController, self)._route(args, request) diff --git a/storyboard/api/v1/tasks.py b/storyboard/api/v1/tasks.py index afb1cfd0..b6b5f5e6 100644 --- a/storyboard/api/v1/tasks.py +++ b/storyboard/api/v1/tasks.py @@ -24,11 +24,14 @@ import wsmeext.pecan as wsme_pecan from storyboard.api.auth import authorization_checks as checks from storyboard.api.v1 import base +from storyboard.api.v1.search import search_engine from storyboard.db.api import tasks as tasks_api from storyboard.db.api import timeline_events as events_api CONF = cfg.CONF +SEARCH_ENGINE = search_engine.get_engine() + class Task(base.APIBase): """A Task represents an actionable work item, targeting a specific Project @@ -69,6 +72,8 @@ class Task(base.APIBase): class TasksController(rest.RestController): """Manages tasks.""" + _custom_actions = {"search": ["GET"]} + @secure(checks.guest) @wsme_pecan.wsexpose(Task, int) def get_one(self, task_id): @@ -222,3 +227,18 @@ class TasksController(rest.RestController): tasks_api.task_delete(task_id) response.status_code = 204 + + @secure(checks.guest) + @wsme_pecan.wsexpose([Task], unicode, unicode, int, int) + def search(self, q="", marker=None, limit=None): + """The search endpoint for tasks. + + :param q: The query string. + :return: List of Tasks matching the query. + """ + + tasks = SEARCH_ENGINE.tasks_query(q=q, + marker=marker, + limit=limit) + + return [Task.from_db_model(task) for task in tasks] diff --git a/storyboard/api/v1/timeline.py b/storyboard/api/v1/timeline.py index 0bca0b26..f5f597d4 100644 --- a/storyboard/api/v1/timeline.py +++ b/storyboard/api/v1/timeline.py @@ -24,6 +24,7 @@ import wsmeext.pecan as wsme_pecan from storyboard.api.auth import authorization_checks as checks from storyboard.api.v1 import base +from storyboard.api.v1.search import search_engine from storyboard.common import event_resolvers from storyboard.common import event_types from storyboard.db.api import comments as comments_api @@ -31,6 +32,8 @@ from storyboard.db.api import timeline_events as events_api CONF = cfg.CONF +SEARCH_ENGINE = search_engine.get_engine() + class Comment(base.APIBase): """Any user may leave comments for stories. Also comments api is used by @@ -300,3 +303,18 @@ class CommentsController(rest.RestController): response.status_code = 204 return response + + @secure(checks.guest) + @wsme_pecan.wsexpose([Comment], unicode, unicode, int, int) + def search(self, q="", marker=None, limit=None): + """The search endpoint for comments. + + :param q: The query string. + :return: List of Comments matching the query. + """ + + comments = SEARCH_ENGINE.comments_query(q=q, + marker=marker, + limit=limit) + + return [Comment.from_db_model(comment) for comment in comments] diff --git a/storyboard/api/v1/users.py b/storyboard/api/v1/users.py index e53ad9a4..35d7eadf 100644 --- a/storyboard/api/v1/users.py +++ b/storyboard/api/v1/users.py @@ -16,6 +16,7 @@ from datetime import datetime from oslo.config import cfg +from pecan import expose from pecan import request from pecan import response from pecan import rest @@ -26,10 +27,13 @@ import wsmeext.pecan as wsme_pecan from storyboard.api.auth import authorization_checks as checks from storyboard.api.v1 import base +from storyboard.api.v1.search import search_engine from storyboard.db.api import users as users_api CONF = cfg.CONF +SEARCH_ENGINE = search_engine.get_engine() + class User(base.APIBase): """Represents a user.""" @@ -69,6 +73,8 @@ class User(base.APIBase): class UsersController(rest.RestController): """Manages users.""" + _custom_actions = {"search": ["GET"]} + @secure(checks.guest) @wsme_pecan.wsexpose([User], int, int, unicode, unicode, unicode, unicode) def get(self, marker=None, limit=None, username=None, full_name=None, @@ -158,3 +164,30 @@ class UsersController(rest.RestController): updated_user = users_api.user_update(user_id, user_dict) return User.from_db_model(updated_user) + + @secure(checks.guest) + @wsme_pecan.wsexpose([User], unicode, unicode, int, int) + def search(self, q="", marker=None, limit=None): + """The search endpoint for users. + + :param q: The query string. + :return: List of Users matching the query. + """ + + users = SEARCH_ENGINE.users_query(q=q, marker=marker, limit=limit) + + return [User.from_db_model(u) for u in users] + + @expose() + def _route(self, args, request): + if request.method == 'GET' and len(args) > 0: + # It's a request by a name or id + something = args[0] + + if something == "search": + # Request to a search endpoint + return self.search, args + else: + return self.get_one, args + + return super(UsersController, self)._route(args, request) diff --git a/storyboard/db/migration/alembic_migrations/versions/022_fulltext_indexes.py b/storyboard/db/migration/alembic_migrations/versions/022_fulltext_indexes.py new file mode 100644 index 00000000..2e230aaf --- /dev/null +++ b/storyboard/db/migration/alembic_migrations/versions/022_fulltext_indexes.py @@ -0,0 +1,71 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Adding full text indexes + +Revision ID: 022 +Revises: 021 +Create Date: 2014-07-11 14:08:08.129484 + +""" + +# revision identifiers, used by Alembic. +revision = '022' +down_revision = '021' + + +from alembic import op + +from storyboard.openstack.common import log + +LOG = log.getLogger(__name__) + + +def upgrade(active_plugins=None, options=None): + + version_info = op.get_bind().engine.dialect.server_version_info + if version_info[0] < 5 or version_info[0] == 5 and version_info[1] < 6: + LOG.warn("MySQL version is lower than 5.6. Skipping full-text indexes") + return + + # Index for projects + op.execute("ALTER TABLE projects " + "ADD FULLTEXT projects_fti (name, description)") + + # Index for stories + op.execute("ALTER TABLE stories " + "ADD FULLTEXT stories_fti (title, description)") + + # Index for tasks + op.execute("ALTER TABLE tasks ADD FULLTEXT tasks_fti (title)") + + # Index for comments + op.execute("ALTER TABLE comments ADD FULLTEXT comments_fti (content)") + + # Index for users + op.execute("ALTER TABLE users " + "ADD FULLTEXT users_fti (username, full_name, email)") + + +def downgrade(active_plugins=None, options=None): + + version_info = op.get_bind().engine.dialect.server_version_info + if version_info[0] < 5 or version_info[0] == 5 and version_info[1] < 6: + LOG.warn("MySQL version is lower than 5.6. Skipping full-text indexes") + return + + op.drop_index("projects_fti", table_name='projects') + op.drop_index("stories_fti", table_name='stories') + op.drop_index("tasks_fti", table_name='tasks') + op.drop_index("comments_fti", table_name='comments') + op.drop_index("users_fti", table_name='users') diff --git a/storyboard/db/models.py b/storyboard/db/models.py index 6937dc71..fafa65a5 100644 --- a/storyboard/db/models.py +++ b/storyboard/db/models.py @@ -36,6 +36,7 @@ from sqlalchemy import String from sqlalchemy import Table from sqlalchemy import Unicode from sqlalchemy import UnicodeText +from sqlalchemy_fulltext import FullText CONF = cfg.CONF @@ -91,11 +92,14 @@ team_membership = Table( ) -class User(Base): +class User(FullText, Base): __table_args__ = ( schema.UniqueConstraint('username', name='uniq_user_username'), schema.UniqueConstraint('email', name='uniq_user_email'), ) + + __fulltext_columns__ = ['username', 'full_name', 'email'] + username = Column(Unicode(30)) full_name = Column(Unicode(255), nullable=True) email = Column(String(255)) @@ -134,13 +138,15 @@ class Permission(Base): # TODO(mordred): Do we really need name and title? -class Project(Base): +class Project(FullText, Base): """Represents a software project.""" __table_args__ = ( schema.UniqueConstraint('name', name='uniq_project_name'), ) + __fulltext_columns__ = ['name', 'description'] + name = Column(String(50)) description = Column(UnicodeText()) team_id = Column(Integer, ForeignKey('teams.id')) @@ -164,9 +170,11 @@ class ProjectGroup(Base): _public_fields = ["id", "name", "title", "projects"] -class Story(Base): +class Story(FullText, Base): __tablename__ = 'stories' + __fulltext_columns__ = ['title', 'description'] + creator_id = Column(Integer, ForeignKey('users.id')) creator = relationship(User, primaryjoin=creator_id == User.id) title = Column(Unicode(100)) @@ -180,7 +188,9 @@ class Story(Base): "tasks", "events", "tags"] -class Task(Base): +class Task(FullText, Base): + __fulltext_columns__ = ['title'] + _TASK_STATUSES = ('todo', 'inprogress', 'invalid', 'review', 'merged') _TASK_PRIORITIES = ('low', 'medium', 'high') @@ -285,7 +295,8 @@ class TimeLineEvent(Base): event_info = Column(UnicodeText(), nullable=True) -class Comment(Base): +class Comment(FullText, Base): + __fulltext_columns__ = ['content'] content = Column(UnicodeText) is_active = Column(Boolean, default=True)