opendev/storyboard
Code Issues Proposed changes

Adding Search endpoints and sqlalchemy impl

Browse Source 
Search endpoints added for Projects, Stories, Tasks, Comments and Users.

SqlAlchemy plugin for fulltext search added.
The migration for full-text indexes added.
The migration checks the MySQL server version before creating indexes.

The default search engine via sqlalchemy added.

Change-Id: Ie3e4c4f317338d68e82c9c21652d49220c6e4a7d
This commit is contained in:
Nikita Konovalov 2014-06-20 14:52:36 +04:00 committed by Michael Krotscheck
parent 810efd226e
commit 860f12ff00
13 changed files with 417 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -14,4 +14,5 @@ six>=1.7.0
SQLAlchemy>=0.8,<=0.8.99
WSME>=0.6
sqlalchemy-migrate>=0.8.2,!=0.8.4
SQLAlchemy-FullText-Search
eventlet>=0.13.0

7
storyboard/api/app.py
View File

@ -24,6 +24,8 @@ from storyboard.api.auth.token_storage import storage
from storyboard.api import config as api_config
from storyboard.api.middleware import token_middleware
from storyboard.api.middleware import user_id_hook
from storyboard.api.v1.search import impls as search_engine_impls
from storyboard.api.v1.search import search_engine
from storyboard.openstack.common.gettextutils import _ # noqa
from storyboard.openstack.common import log
@ -65,6 +67,11 @@ def setup_app(pecan_config=None):
storage_cls = storage_impls.STORAGE_IMPLS[token_storage_type]
storage.set_storage(storage_cls())
# Setup search engine
search_engine_name = CONF.search_engine
search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name]
search_engine.set_engine(search_engine_cls())
app = pecan.make_app(
pecan_config.app.root,
debug=CONF.debug,

28
storyboard/api/v1/projects.py
View File

@ -24,11 +24,14 @@ import wsmeext.pecan as wsme_pecan
from storyboard.api.auth import authorization_checks as checks
from storyboard.api.v1 import base
from storyboard.api.v1.search import search_engine
from storyboard.common.custom_types import NameType
from storyboard.db.api import projects as projects_api
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class Project(base.APIBase):
"""The Storyboard Registry describes the open source world as ProjectGroups
@ -66,6 +69,8 @@ class ProjectsController(rest.RestController):
At this moment it provides read-only operations.
"""
_custom_actions = {"search": ["GET"]}
@secure(checks.guest)
@wsme_pecan.wsexpose(Project, int)
def get_one_by_id(self, project_id):
@ -171,12 +176,31 @@ class ProjectsController(rest.RestController):
except ValueError:
return False
@secure(checks.guest)
@wsme_pecan.wsexpose([Project], unicode, unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for projects.
:param q: The query string.
:return: List of Projects matching the query.
"""
projects = SEARCH_ENGINE.projects_query(q=q, marker=marker,
limit=limit)
return [Project.from_db_model(project) for project in projects]
@expose()
def _route(self, args, request):
if request.method == 'GET' and len(args) > 0:
# It's a request by a name or id
first_token = args[0]
if self._is_int(first_token):
something = args[0]
if something == "search":
# Request to a search endpoint
return super(ProjectsController, self)._route(args, request)
if self._is_int(something):
# Get by id
return self.get_one_by_id, args
else:

0
storyboard/api/v1/search/__init__.py Normal file
View File

21
storyboard/api/v1/search/impls.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from storyboard.api.v1.search.sqlalchemy_impl import SqlAlchemySearchImpl
ENGINE_IMPLS = {
"sqlalchemy": SqlAlchemySearchImpl
}

88
storyboard/api/v1/search/search_engine.py Normal file
View File

@ -0,0 +1,88 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
from oslo.config import cfg
from storyboard.db import models
CONF = cfg.CONF
SEARCH_OPTS = [
cfg.StrOpt('search_engine',
default='sqlalchemy',
help='Search engine implementation.'
' The only supported type is "sqlalchemy".')
]
CONF.register_opts(SEARCH_OPTS)
class SearchEngine(object):
"""This is an interface that should be implemented by search engines.
"""
searchable_fields = {
models.Project: ["name", "description"],
models.Story: ["title", "description"],
models.Task: ["title"],
models.Comment: ["content"],
models.User: ['username', 'full_name', 'email']
}
@abc.abstractmethod
def projects_query(self, q, sort_dir=None, marker=None, limit=None,
**kwargs):
pass
@abc.abstractmethod
def stories_query(self, q, status=None, author=None,
created_after=None, created_before=None,
updated_after=None, updated_before=None,
marker=None, limit=None, **kwargs):
pass
@abc.abstractmethod
def tasks_query(self, q, status=None, author=None, priority=None,
assignee=None, project=None, project_group=None,
created_after=None, created_before=None,
updated_after=None, updated_before=None,
marker=None, limit=None, **kwargs):
pass
@abc.abstractmethod
def comments_query(self, q, created_after=None, created_before=None,
updated_after=None, updated_before=None,
marker=None, limit=None, **kwargs):
pass
@abc.abstractmethod
def users_query(self, q, marker=None, limit=None, **kwargs):
pass
ENGINE = None
def get_engine():
global ENGINE
return ENGINE
def set_engine(impl):
global ENGINE
ENGINE = impl

83
storyboard/api/v1/search/sqlalchemy_impl.py Normal file
View File

@ -0,0 +1,83 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.db.sqlalchemy import utils
from sqlalchemy_fulltext import FullTextSearch
import sqlalchemy_fulltext.modes as FullTextMode
from storyboard.api.v1.search import search_engine
from storyboard.db.api import base as api_base
from storyboard.db import models
class SqlAlchemySearchImpl(search_engine.SearchEngine):
def _build_fulltext_search(self, model_cls, query, q):
query = query.filter(FullTextSearch(q, model_cls,
mode=FullTextMode.NATURAL))
return query
def _apply_pagination(self, model_cls, query, marker=None, limit=None):
marker_entity = None
if marker:
marker_entity = api_base.entity_get(model_cls, marker, True)
return utils.paginate_query(query=query,
model=model_cls,
limit=limit,
sort_keys=["id"],
marker=marker_entity)
def projects_query(self, q, sort_dir=None, marker=None, limit=None):
session = api_base.get_session()
query = api_base.model_query(models.Project, session)
query = self._build_fulltext_search(models.Project, query, q)
query = self._apply_pagination(models.Project, query, marker, limit)
return query.all()
def stories_query(self, q, marker=None, limit=None, **kwargs):
session = api_base.get_session()
query = api_base.model_query(models.Story, session)
query = self._build_fulltext_search(models.Story, query, q)
query = self._apply_pagination(models.Story, query, marker, limit)
return query.all()
def tasks_query(self, q, marker=None, limit=None, **kwargs):
session = api_base.get_session()
query = api_base.model_query(models.Task, session)
query = self._build_fulltext_search(models.Task, query, q)
query = self._apply_pagination(models.Task, query, marker, limit)
return query.all()
def comments_query(self, q, marker=None, limit=None, **kwargs):
session = api_base.get_session()
query = api_base.model_query(models.Comment, session)
query = self._build_fulltext_search(models.Comment, query, q)
query = self._apply_pagination(models.Comment, query, marker, limit)
return query.all()
def users_query(self, q, marker=None, limit=None, **kwargs):
session = api_base.get_session()
query = api_base.model_query(models.User, session)
query = self._build_fulltext_search(models.User, query, q)
query = self._apply_pagination(models.User, query, marker, limit)
return query.all()

33
storyboard/api/v1/stories.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
from oslo.config import cfg
from pecan import expose
from pecan import request
from pecan import response
from pecan import rest
@ -24,6 +25,7 @@ import wsmeext.pecan as wsme_pecan
from storyboard.api.auth import authorization_checks as checks
from storyboard.api.v1 import base
from storyboard.api.v1.search import search_engine
from storyboard.api.v1.timeline import CommentsController
from storyboard.api.v1.timeline import TimeLineEventsController
from storyboard.db.api import stories as stories_api
@ -31,6 +33,8 @@ from storyboard.db.api import timeline_events as events_api
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class Story(base.APIBase):
"""The Story is the main element of StoryBoard. It represents a user story
@ -87,6 +91,8 @@ class Story(base.APIBase):
class StoriesController(rest.RestController):
"""Manages operations on stories."""
_custom_actions = {"search": ["GET"]}
@secure(checks.guest)
@wsme_pecan.wsexpose(Story, int)
def get_one(self, story_id):
@ -214,3 +220,30 @@ class StoriesController(rest.RestController):
comments = CommentsController()
events = TimeLineEventsController()
@secure(checks.guest)
@wsme_pecan.wsexpose([Story], unicode, unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for stories.
:param q: The query string.
:return: List of Stories matching the query.
"""
stories = SEARCH_ENGINE.stories_query(q=q,
marker=marker,
limit=limit)
return [Story.from_db_model(story) for story in stories]
@expose()
def _route(self, args, request):
if request.method == 'GET' and len(args) > 0:
# It's a request by a name or id
something = args[0]
if something == "search":
# Request to a search endpoint
return self.search, args
return super(StoriesController, self)._route(args, request)

20
storyboard/api/v1/tasks.py
View File

@ -24,11 +24,14 @@ import wsmeext.pecan as wsme_pecan
from storyboard.api.auth import authorization_checks as checks
from storyboard.api.v1 import base
from storyboard.api.v1.search import search_engine
from storyboard.db.api import tasks as tasks_api
from storyboard.db.api import timeline_events as events_api
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class Task(base.APIBase):
"""A Task represents an actionable work item, targeting a specific Project
@ -69,6 +72,8 @@ class Task(base.APIBase):
class TasksController(rest.RestController):
"""Manages tasks."""
_custom_actions = {"search": ["GET"]}
@secure(checks.guest)
@wsme_pecan.wsexpose(Task, int)
def get_one(self, task_id):
@ -222,3 +227,18 @@ class TasksController(rest.RestController):
tasks_api.task_delete(task_id)
response.status_code = 204
@secure(checks.guest)
@wsme_pecan.wsexpose([Task], unicode, unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for tasks.
:param q: The query string.
:return: List of Tasks matching the query.
"""
tasks = SEARCH_ENGINE.tasks_query(q=q,
marker=marker,
limit=limit)
return [Task.from_db_model(task) for task in tasks]

18
storyboard/api/v1/timeline.py
View File

@ -24,6 +24,7 @@ import wsmeext.pecan as wsme_pecan
from storyboard.api.auth import authorization_checks as checks
from storyboard.api.v1 import base
from storyboard.api.v1.search import search_engine
from storyboard.common import event_resolvers
from storyboard.common import event_types
from storyboard.db.api import comments as comments_api
@ -31,6 +32,8 @@ from storyboard.db.api import timeline_events as events_api
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class Comment(base.APIBase):
"""Any user may leave comments for stories. Also comments api is used by
@ -300,3 +303,18 @@ class CommentsController(rest.RestController):
response.status_code = 204
return response
@secure(checks.guest)
@wsme_pecan.wsexpose([Comment], unicode, unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for comments.
:param q: The query string.
:return: List of Comments matching the query.
"""
comments = SEARCH_ENGINE.comments_query(q=q,
marker=marker,
limit=limit)
return [Comment.from_db_model(comment) for comment in comments]

33
storyboard/api/v1/users.py
View File

@ -16,6 +16,7 @@
from datetime import datetime
from oslo.config import cfg
from pecan import expose
from pecan import request
from pecan import response
from pecan import rest
@ -26,10 +27,13 @@ import wsmeext.pecan as wsme_pecan
from storyboard.api.auth import authorization_checks as checks
from storyboard.api.v1 import base
from storyboard.api.v1.search import search_engine
from storyboard.db.api import users as users_api
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class User(base.APIBase):
"""Represents a user."""
@ -69,6 +73,8 @@ class User(base.APIBase):
class UsersController(rest.RestController):
"""Manages users."""
_custom_actions = {"search": ["GET"]}
@secure(checks.guest)
@wsme_pecan.wsexpose([User], int, int, unicode, unicode, unicode, unicode)
def get(self, marker=None, limit=None, username=None, full_name=None,
@ -158,3 +164,30 @@ class UsersController(rest.RestController):
updated_user = users_api.user_update(user_id, user_dict)
return User.from_db_model(updated_user)
@secure(checks.guest)
@wsme_pecan.wsexpose([User], unicode, unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for users.
:param q: The query string.
:return: List of Users matching the query.
"""
users = SEARCH_ENGINE.users_query(q=q, marker=marker, limit=limit)
return [User.from_db_model(u) for u in users]
@expose()
def _route(self, args, request):
if request.method == 'GET' and len(args) > 0:
# It's a request by a name or id
something = args[0]
if something == "search":
# Request to a search endpoint
return self.search, args
else:
return self.get_one, args
return super(UsersController, self)._route(args, request)

71
storyboard/db/migration/alembic_migrations/versions/022_fulltext_indexes.py Normal file
View File

@ -0,0 +1,71 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""Adding full text indexes
Revision ID: 022
Revises: 021
Create Date: 2014-07-11 14:08:08.129484
"""
# revision identifiers, used by Alembic.
revision = '022'
down_revision = '021'
from alembic import op
from storyboard.openstack.common import log
LOG = log.getLogger(__name__)
def upgrade(active_plugins=None, options=None):
version_info = op.get_bind().engine.dialect.server_version_info
if version_info[0] < 5 or version_info[0] == 5 and version_info[1] < 6:
LOG.warn("MySQL version is lower than 5.6. Skipping full-text indexes")
return
# Index for projects
op.execute("ALTER TABLE projects "
"ADD FULLTEXT projects_fti (name, description)")
# Index for stories
op.execute("ALTER TABLE stories "
"ADD FULLTEXT stories_fti (title, description)")
# Index for tasks
op.execute("ALTER TABLE tasks ADD FULLTEXT tasks_fti (title)")
# Index for comments
op.execute("ALTER TABLE comments ADD FULLTEXT comments_fti (content)")
# Index for users
op.execute("ALTER TABLE users "
"ADD FULLTEXT users_fti (username, full_name, email)")
def downgrade(active_plugins=None, options=None):
version_info = op.get_bind().engine.dialect.server_version_info
if version_info[0] < 5 or version_info[0] == 5 and version_info[1] < 6:
LOG.warn("MySQL version is lower than 5.6. Skipping full-text indexes")
return
op.drop_index("projects_fti", table_name='projects')
op.drop_index("stories_fti", table_name='stories')
op.drop_index("tasks_fti", table_name='tasks')
op.drop_index("comments_fti", table_name='comments')
op.drop_index("users_fti", table_name='users')

21
storyboard/db/models.py
View File

@ -36,6 +36,7 @@ from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import Unicode
from sqlalchemy import UnicodeText
from sqlalchemy_fulltext import FullText
CONF = cfg.CONF
@ -91,11 +92,14 @@ team_membership = Table(
)
class User(Base):
class User(FullText, Base):
__table_args__ = (
schema.UniqueConstraint('username', name='uniq_user_username'),
schema.UniqueConstraint('email', name='uniq_user_email'),
)
__fulltext_columns__ = ['username', 'full_name', 'email']
username = Column(Unicode(30))
full_name = Column(Unicode(255), nullable=True)
email = Column(String(255))
@ -134,13 +138,15 @@ class Permission(Base):
# TODO(mordred): Do we really need name and title?
class Project(Base):
class Project(FullText, Base):
"""Represents a software project."""
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_project_name'),
)
__fulltext_columns__ = ['name', 'description']
name = Column(String(50))
description = Column(UnicodeText())
team_id = Column(Integer, ForeignKey('teams.id'))
@ -164,9 +170,11 @@ class ProjectGroup(Base):
_public_fields = ["id", "name", "title", "projects"]
class Story(Base):
class Story(FullText, Base):
__tablename__ = 'stories'
__fulltext_columns__ = ['title', 'description']
creator_id = Column(Integer, ForeignKey('users.id'))
creator = relationship(User, primaryjoin=creator_id == User.id)
title = Column(Unicode(100))
@ -180,7 +188,9 @@ class Story(Base):
"tasks", "events", "tags"]
class Task(Base):
class Task(FullText, Base):
__fulltext_columns__ = ['title']
_TASK_STATUSES = ('todo', 'inprogress', 'invalid', 'review', 'merged')
_TASK_PRIORITIES = ('low', 'medium', 'high')
@ -285,7 +295,8 @@ class TimeLineEvent(Base):
event_info = Column(UnicodeText(), nullable=True)
class Comment(Base):
class Comment(FullText, Base):
__fulltext_columns__ = ['content']
content = Column(UnicodeText)
is_active = Column(Boolean, default=True)