Implemented caching for API calls

Introduced new decorators:
 * response - responsible for parameters processing
 * cached - get/set to cache
Removed all search functionality in API methods.
Fixed logging.

implements bp cache-api-methods

Change-Id: I91d36424a36442a09591c140e9f22401139b7415
This commit is contained in:
Ilya Shakhat 2014-05-21 14:32:34 +04:00
parent 68ecbf6db1
commit f36fd65ab2
10 changed files with 235 additions and 202 deletions

View File

@ -32,13 +32,67 @@ from stackalytics import version as stackalytics_version
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
def _filter_records_by_days(ignore, start_date, end_date, memory_storage_inst): def _prepare_params(kwargs, ignore):
if start_date and 'start_date' not in ignore: params = kwargs.get('_params')
start_date = utils.date_to_timestamp_ext(start_date)
if not params:
params = {'action': flask.request.path}
for key in parameters.FILTER_PARAMETERS:
params[key] = parameters.get_parameter(kwargs, key, key)
if params['start_date']:
params['start_date'] = [utils.round_timestamp_to_day(
params['start_date'][0])]
if params['end_date']:
params['end_date'] = [utils.round_timestamp_to_day(
params['end_date'][0])]
kwargs['_params'] = params
if ignore:
return dict([(k, v if k not in ignore else [])
for k, v in six.iteritems(params)])
else:
return params
def cached(ignore=None):
def decorator(func):
@functools.wraps(func)
def prepare_params_decorated_function(*args, **kwargs):
params = _prepare_params(kwargs, ignore)
cache_inst = vault.get_vault()['cache']
key = json.dumps(params)
value = cache_inst.get(key)
if not value:
value = func(*args, **kwargs)
cache_inst[key] = value
vault.get_vault()['cache_size'] += len(key) + len(value)
LOG.debug('Cache size: %(size)d, entries: %(len)d',
{'size': vault.get_vault()['cache_size'],
'len': len(cache_inst.keys())})
return value
return prepare_params_decorated_function
return decorator
def record_filter(ignore=None):
def decorator(f):
def _filter_records_by_days(start_date, end_date, memory_storage_inst):
if start_date:
start_date = utils.date_to_timestamp_ext(start_date[0])
else: else:
start_date = memory_storage_inst.get_first_record_day() start_date = memory_storage_inst.get_first_record_day()
if end_date and 'end_date' not in ignore: if end_date:
end_date = utils.date_to_timestamp_ext(end_date) end_date = utils.date_to_timestamp_ext(end_date[0])
else: else:
end_date = utils.date_to_timestamp_ext('now') end_date = utils.date_to_timestamp_ext('now')
@ -48,12 +102,6 @@ def _filter_records_by_days(ignore, start_date, end_date, memory_storage_inst):
return memory_storage_inst.get_record_ids_by_days( return memory_storage_inst.get_record_ids_by_days(
six.moves.range(start_day, end_day + 1)) six.moves.range(start_day, end_day + 1))
def record_filter(ignore=None, use_default=True):
if not ignore:
ignore = []
def decorator(f):
def _filter_records_by_modules(memory_storage_inst, modules, releases): def _filter_records_by_modules(memory_storage_inst, modules, releases):
selected = set([]) selected = set([])
for m, r in vault.resolve_modules(modules, releases): for m, r in vault.resolve_modules(modules, releases):
@ -71,58 +119,47 @@ def record_filter(ignore=None, use_default=True):
memory_storage_inst = vault.get_memory_storage() memory_storage_inst = vault.get_memory_storage()
record_ids = set(memory_storage_inst.get_record_ids()) # a copy record_ids = set(memory_storage_inst.get_record_ids()) # a copy
releases = [] params = _prepare_params(kwargs, ignore)
if 'release' not in ignore:
releases = parameters.get_parameter(kwargs, 'release', release = params['release']
'releases', use_default) if release:
if releases: if 'all' not in release:
if 'all' not in releases:
record_ids &= ( record_ids &= (
memory_storage_inst.get_record_ids_by_releases( memory_storage_inst.get_record_ids_by_releases(
c.lower() for c in releases)) c.lower() for c in release))
modules = parameters.get_parameter(kwargs, 'module', 'modules', project_type = params['project_type']
use_default) if project_type:
if 'project_type' not in ignore:
param = parameters.get_parameter(kwargs, 'project_type',
'project_types', use_default)
if param:
record_ids &= _filter_records_by_modules( record_ids &= _filter_records_by_modules(
memory_storage_inst, memory_storage_inst,
vault.resolve_project_types(param), vault.resolve_project_types(project_type), release)
releases)
if 'module' not in ignore: module = params['module']
if modules: if module:
record_ids &= _filter_records_by_modules( record_ids &= _filter_records_by_modules(
memory_storage_inst, modules, releases) memory_storage_inst, module, release)
if 'user_id' not in ignore: user_id = params['user_id']
param = parameters.get_parameter(kwargs, 'user_id', 'user_ids') user_id = [u for u in user_id
param = [u for u in param
if vault.get_user_from_runtime_storage(u)] if vault.get_user_from_runtime_storage(u)]
if param: if user_id:
record_ids &= ( record_ids &= (
memory_storage_inst.get_record_ids_by_user_ids(param)) memory_storage_inst.get_record_ids_by_user_ids(user_id))
if 'company' not in ignore: company = params['company']
param = parameters.get_parameter(kwargs, 'company', if company:
'companies')
if param:
record_ids &= ( record_ids &= (
memory_storage_inst.get_record_ids_by_companies(param)) memory_storage_inst.get_record_ids_by_companies(company))
if 'metric' not in ignore: metric = params['metric']
metrics = parameters.get_parameter(kwargs, 'metric') if 'all' not in metric:
if 'all' not in metrics: for metric in metric:
for metric in metrics:
if metric in parameters.METRIC_TO_RECORD_TYPE: if metric in parameters.METRIC_TO_RECORD_TYPE:
record_ids &= ( record_ids &= (
memory_storage_inst.get_record_ids_by_type( memory_storage_inst.get_record_ids_by_type(
parameters.METRIC_TO_RECORD_TYPE[metric])) parameters.METRIC_TO_RECORD_TYPE[metric]))
if 'tm_marks' in metrics: if 'tm_marks' in metric:
filtered_ids = [] filtered_ids = []
review_nth = int(parameters.get_parameter( review_nth = int(parameters.get_parameter(
kwargs, 'review_nth')[0]) kwargs, 'review_nth')[0])
@ -134,24 +171,22 @@ def record_filter(ignore=None, use_default=True):
filtered_ids.append(record['record_id']) filtered_ids.append(record['record_id'])
record_ids = filtered_ids record_ids = filtered_ids
if 'blueprint_id' not in ignore: blueprint_id = params['blueprint_id']
param = parameters.get_parameter(kwargs, 'blueprint_id') if blueprint_id:
if param:
record_ids &= ( record_ids &= (
memory_storage_inst.get_record_ids_by_blueprint_ids( memory_storage_inst.get_record_ids_by_blueprint_ids(
param)) blueprint_id))
start_date = parameters.get_single_parameter(kwargs, 'start_date') start_date = params['start_date']
end_date = parameters.get_single_parameter(kwargs, 'end_date') end_date = params['end_date']
if (start_date and 'start_date' not in ignore) or ( if start_date or end_date:
end_date and 'end_date' not in ignore): record_ids &= _filter_records_by_days(start_date, end_date,
record_ids &= _filter_records_by_days(ignore,
start_date, end_date,
memory_storage_inst) memory_storage_inst)
kwargs['record_ids'] = record_ids kwargs['record_ids'] = record_ids
kwargs['records'] = memory_storage_inst.get_records(record_ids) kwargs['records'] = memory_storage_inst.get_records(record_ids)
return f(*args, **kwargs) return f(*args, **kwargs)
return record_filter_decorated_function return record_filter_decorated_function
@ -362,8 +397,19 @@ def jsonify(root='data'):
def decorator(func): def decorator(func):
@functools.wraps(func) @functools.wraps(func)
def jsonify_decorated_function(*args, **kwargs): def jsonify_decorated_function(*args, **kwargs):
return json.dumps({root: func(*args, **kwargs)})
return jsonify_decorated_function
return decorator
def response():
def decorator(func):
@functools.wraps(func)
def response_decorated_function(*args, **kwargs):
callback = flask.app.request.args.get('callback', False) callback = flask.app.request.args.get('callback', False)
data = json.dumps({root: func(*args, **kwargs)}) data = func(*args, **kwargs)
if callback: if callback:
data = str(callback) + '(' + data + ')' data = str(callback) + '(' + data + ')'
@ -373,7 +419,7 @@ def jsonify(root='data'):
return flask.current_app.response_class(data, mimetype=mimetype) return flask.current_app.response_class(data, mimetype=mimetype)
return jsonify_decorated_function return response_decorated_function
return decorator return decorator

View File

@ -48,6 +48,9 @@ METRIC_TO_RECORD_TYPE = {
'members': 'member', 'members': 'member',
} }
FILTER_PARAMETERS = ['release', 'project_type', 'module', 'company', 'user_id',
'metric', 'start_date', 'end_date', 'blueprint_id']
DEFAULT_RECORDS_LIMIT = 10 DEFAULT_RECORDS_LIMIT = 10
DEFAULT_STATIC_ACTIVITY_SIZE = 100 DEFAULT_STATIC_ACTIVITY_SIZE = 100

View File

@ -223,6 +223,7 @@ def activity():
@blueprint.route('/large_commits') @blueprint.route('/large_commits')
@decorators.response()
@decorators.jsonify('commits') @decorators.jsonify('commits')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter() @decorators.record_filter()
@ -240,6 +241,7 @@ def get_commit_report(records, **kwargs):
@blueprint.route('/single_plus_two_reviews') @blueprint.route('/single_plus_two_reviews')
@decorators.response()
@decorators.jsonify() @decorators.jsonify()
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter(ignore='metric') @decorators.record_filter(ignore='metric')

View File

@ -62,8 +62,6 @@ def get_vault():
vault['memory_storage'] = memory_storage.get_memory_storage( vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED) memory_storage.MEMORY_STORAGE_CACHED)
_init_releases(vault)
flask.current_app.stackalytics_vault = vault flask.current_app.stackalytics_vault = vault
except Exception as e: except Exception as e:
LOG.critical('Failed to initialize application: %s', e) LOG.critical('Failed to initialize application: %s', e)
@ -77,6 +75,8 @@ def get_vault():
compact_records(vault['runtime_storage'].get_update(os.getpid()))) compact_records(vault['runtime_storage'].get_update(os.getpid())))
if have_updates: if have_updates:
vault['cache'] = {}
vault['cache_size'] = 0
_init_releases(vault) _init_releases(vault)
_init_module_groups(vault) _init_module_groups(vault)
_init_project_types(vault) _init_project_types(vault)

View File

@ -46,15 +46,14 @@ LOG = logging.getLogger(__name__)
conf = cfg.CONF conf = cfg.CONF
conf.register_opts(config.OPTS) conf.register_opts(config.OPTS)
logging.setup('dashboard')
LOG.info('Logging enabled')
conf_file = os.getenv('STACKALYTICS_CONF') conf_file = os.getenv('STACKALYTICS_CONF')
if conf_file and os.path.isfile(conf_file): if conf_file and os.path.isfile(conf_file):
conf(default_config_files=[conf_file]) conf(default_config_files=[conf_file])
app.config['DEBUG'] = cfg.CONF.debug app.config['DEBUG'] = cfg.CONF.debug
else:
LOG.info('Conf file is empty or not exist') logging.setup('dashboard')
LOG.info('Stackalytics.dashboard is configured via "%s"', conf_file)
# Handlers --------- # Handlers ---------
@ -94,9 +93,10 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id,
@app.route('/api/1.0/new_companies') @app.route('/api/1.0/new_companies')
@decorators.jsonify('stats')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter(ignore='start_date') @decorators.response()
@decorators.jsonify('stats')
@decorators.record_filter(ignore=['start_date'])
def get_new_companies(records, **kwargs): def get_new_companies(records, **kwargs):
days = int(flask.request.args.get('days') or reports.DEFAULT_DAYS_COUNT) days = int(flask.request.args.get('days') or reports.DEFAULT_DAYS_COUNT)
@ -123,8 +123,10 @@ def get_new_companies(records, **kwargs):
@app.route('/api/1.0/stats/companies') @app.route('/api/1.0/stats/companies')
@decorators.jsonify('stats')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.cached()
@decorators.jsonify('stats')
@decorators.record_filter() @decorators.record_filter()
@decorators.aggregate_filter() @decorators.aggregate_filter()
def get_companies(records, metric_filter, finalize_handler, **kwargs): def get_companies(records, metric_filter, finalize_handler, **kwargs):
@ -135,8 +137,10 @@ def get_companies(records, metric_filter, finalize_handler, **kwargs):
@app.route('/api/1.0/stats/modules') @app.route('/api/1.0/stats/modules')
@decorators.jsonify('stats')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.cached()
@decorators.jsonify('stats')
@decorators.record_filter() @decorators.record_filter()
@decorators.aggregate_filter() @decorators.aggregate_filter()
def get_modules(records, metric_filter, finalize_handler, **kwargs): def get_modules(records, metric_filter, finalize_handler, **kwargs):
@ -156,8 +160,10 @@ def get_core_engineer_branch(user, modules):
@app.route('/api/1.0/stats/engineers') @app.route('/api/1.0/stats/engineers')
@decorators.jsonify('stats')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.cached()
@decorators.jsonify('stats')
@decorators.record_filter() @decorators.record_filter()
@decorators.aggregate_filter() @decorators.aggregate_filter()
def get_engineers(records, metric_filter, finalize_handler, **kwargs): def get_engineers(records, metric_filter, finalize_handler, **kwargs):
@ -179,9 +185,11 @@ def get_engineers(records, metric_filter, finalize_handler, **kwargs):
@app.route('/api/1.0/stats/engineers_extended') @app.route('/api/1.0/stats/engineers_extended')
@decorators.jsonify('stats')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter(ignore='metric') @decorators.response()
@decorators.cached(ignore=['metric'])
@decorators.jsonify('stats')
@decorators.record_filter(ignore=['metric'])
def get_engineers_extended(records, **kwargs): def get_engineers_extended(records, **kwargs):
modules_names = parameters.get_parameter({}, 'module', 'modules') modules_names = parameters.get_parameter({}, 'module', 'modules')
modules = set([m for m, r in vault.resolve_modules(modules_names, [''])]) modules = set([m for m, r in vault.resolve_modules(modules_names, [''])])
@ -224,8 +232,10 @@ def get_engineers_extended(records, **kwargs):
@app.route('/api/1.0/stats/distinct_engineers') @app.route('/api/1.0/stats/distinct_engineers')
@decorators.jsonify('stats')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.cached()
@decorators.jsonify('stats')
@decorators.record_filter() @decorators.record_filter()
def get_distinct_engineers(records, **kwargs): def get_distinct_engineers(records, **kwargs):
result = {} result = {}
@ -238,8 +248,9 @@ def get_distinct_engineers(records, **kwargs):
@app.route('/api/1.0/activity') @app.route('/api/1.0/activity')
@decorators.jsonify('activity')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.jsonify('activity')
@decorators.record_filter() @decorators.record_filter()
def get_activity_json(records, **kwargs): def get_activity_json(records, **kwargs):
start_record = int(flask.request.args.get('start_record') or 0) start_record = int(flask.request.args.get('start_record') or 0)
@ -251,38 +262,40 @@ def get_activity_json(records, **kwargs):
@app.route('/api/1.0/contribution') @app.route('/api/1.0/contribution')
@decorators.jsonify('contribution')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter(ignore='metric') @decorators.response()
@decorators.cached(ignore=['metric'])
@decorators.jsonify('contribution')
@decorators.record_filter(ignore=['metric'])
def get_contribution_json(records, **kwargs): def get_contribution_json(records, **kwargs):
return helpers.get_contribution_summary(records) return helpers.get_contribution_summary(records)
@app.route('/api/1.0/companies') @app.route('/api/1.0/companies')
@decorators.jsonify('companies')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter(ignore='company') @decorators.response()
@decorators.query_filter(query_param='company_name') @decorators.cached(ignore=['company'])
def get_companies_json(record_ids, query_filter, **kwargs): @decorators.jsonify('companies')
@decorators.record_filter(ignore=['company'])
def get_companies_json(record_ids, **kwargs):
memory_storage = vault.get_memory_storage() memory_storage = vault.get_memory_storage()
companies = memory_storage.get_index_keys_by_record_ids( companies = memory_storage.get_index_keys_by_record_ids(
'company_name', record_ids) 'company_name', record_ids)
result = [] result = [memory_storage.get_original_company_name(company)
for company in companies: for company in companies]
if query_filter(company):
result.append(memory_storage.get_original_company_name(company))
return [{'id': utils.safe_encode(c.lower()), 'text': c} return [{'id': utils.safe_encode(c.lower()), 'text': c}
for c in sorted(result)] for c in sorted(result)]
@app.route('/api/1.0/modules') @app.route('/api/1.0/modules')
@decorators.jsonify('modules')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter(ignore='module') @decorators.response()
@decorators.query_filter(query_param='query') @decorators.cached(ignore=['module'])
def get_modules_json(record_ids, query_filter, **kwargs): @decorators.jsonify('modules')
@decorators.record_filter(ignore=['module'])
def get_modules_json(record_ids, **kwargs):
module_id_index = vault.get_vault()['module_id_index'] module_id_index = vault.get_vault()['module_id_index']
tags = parameters.get_parameter({}, 'tag', 'tags') tags = parameters.get_parameter({}, 'tag', 'tags')
@ -305,7 +318,6 @@ def get_modules_json(record_ids, query_filter, **kwargs):
result = [] result = []
for module_id in module_ids: for module_id in module_ids:
module = module_id_index[module_id] module = module_id_index[module_id]
if query_filter(module['module_group_name']):
result.append({'id': module['id'], result.append({'id': module['id'],
'text': module['module_group_name'], 'text': module['module_group_name'],
'tag': module['tag']}) 'tag': module['tag']})
@ -314,6 +326,7 @@ def get_modules_json(record_ids, query_filter, **kwargs):
@app.route('/api/1.0/companies/<company_name>') @app.route('/api/1.0/companies/<company_name>')
@decorators.response()
@decorators.jsonify('company') @decorators.jsonify('company')
def get_company(company_name): def get_company(company_name):
memory_storage_inst = vault.get_memory_storage() memory_storage_inst = vault.get_memory_storage()
@ -328,6 +341,7 @@ def get_company(company_name):
@app.route('/api/1.0/modules/<module>') @app.route('/api/1.0/modules/<module>')
@decorators.response()
@decorators.jsonify('module') @decorators.jsonify('module')
def get_module(module): def get_module(module):
module_id_index = vault.get_vault()['module_id_index'] module_id_index = vault.get_vault()['module_id_index']
@ -340,8 +354,10 @@ def get_module(module):
@app.route('/api/1.0/members') @app.route('/api/1.0/members')
@decorators.jsonify('members')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.cached(ignore=['release', 'project_type', 'module'])
@decorators.jsonify('members')
@decorators.record_filter(ignore=['release', 'project_type', 'module']) @decorators.record_filter(ignore=['release', 'project_type', 'module'])
def get_members(records, **kwargs): def get_members(records, **kwargs):
response = [] response = []
@ -359,8 +375,10 @@ def get_members(records, **kwargs):
@app.route('/api/1.0/stats/bp') @app.route('/api/1.0/stats/bp')
@decorators.jsonify('stats')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.cached()
@decorators.jsonify('stats')
@decorators.record_filter() @decorators.record_filter()
def get_bpd(records, **kwargs): def get_bpd(records, **kwargs):
result = [] result = []
@ -389,25 +407,26 @@ def get_bpd(records, **kwargs):
@app.route('/api/1.0/users') @app.route('/api/1.0/users')
@decorators.jsonify('users')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.record_filter(ignore='user_id') @decorators.response()
@decorators.query_filter(query_param='user_name') @decorators.cached(ignore=['user_id'])
def get_users_json(record_ids, query_filter, **kwargs): @decorators.jsonify('users')
@decorators.record_filter(ignore=['user_id'])
def get_users_json(record_ids, **kwargs):
user_ids = vault.get_memory_storage().get_index_keys_by_record_ids( user_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
'user_id', record_ids) 'user_id', record_ids)
result = [] result = [{'id': user_id,
for user_id in user_ids: 'text': (vault.get_user_from_runtime_storage(user_id)
user_name = vault.get_user_from_runtime_storage(user_id)['user_name'] ['user_name'])}
if query_filter(user_name): for user_id in user_ids]
result.append({'id': user_id, 'text': user_name})
result.sort(key=lambda x: x['text']) result.sort(key=lambda x: x['text'])
return result return result
@app.route('/api/1.0/users/<user_id>') @app.route('/api/1.0/users/<user_id>')
@decorators.response()
@decorators.jsonify('user') @decorators.jsonify('user')
def get_user(user_id): def get_user(user_id):
user = vault.get_user_from_runtime_storage(user_id) user = vault.get_user_from_runtime_storage(user_id)
@ -418,16 +437,17 @@ def get_user(user_id):
@app.route('/api/1.0/releases') @app.route('/api/1.0/releases')
@decorators.jsonify('releases')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.query_filter(query_param='query') @decorators.response()
def get_releases_json(query_filter): @decorators.cached(ignore=parameters.FILTER_PARAMETERS)
@decorators.jsonify('releases')
def get_releases_json(**kwargs):
return [{'id': r['release_name'], 'text': r['release_name'].capitalize()} return [{'id': r['release_name'], 'text': r['release_name'].capitalize()}
for r in vault.get_release_options() for r in vault.get_release_options()]
if query_filter(r['release_name'])]
@app.route('/api/1.0/releases/<release>') @app.route('/api/1.0/releases/<release>')
@decorators.response()
@decorators.jsonify('release') @decorators.jsonify('release')
def get_release_json(release): def get_release_json(release):
if release != 'all': if release != 'all':
@ -438,17 +458,18 @@ def get_release_json(release):
@app.route('/api/1.0/metrics') @app.route('/api/1.0/metrics')
@decorators.jsonify('metrics')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.query_filter(query_param='query') @decorators.response()
def get_metrics_json(query_filter): @decorators.cached(ignore=parameters.FILTER_PARAMETERS)
@decorators.jsonify('metrics')
def get_metrics_json(**kwargs):
return sorted([{'id': m, 'text': t} return sorted([{'id': m, 'text': t}
for m, t in six.iteritems(parameters.METRIC_LABELS) for m, t in six.iteritems(parameters.METRIC_LABELS)],
if query_filter(t)],
key=operator.itemgetter('text')) key=operator.itemgetter('text'))
@app.route('/api/1.0/metrics/<metric>') @app.route('/api/1.0/metrics/<metric>')
@decorators.response()
@decorators.jsonify('metric') @decorators.jsonify('metric')
@decorators.exception_handler() @decorators.exception_handler()
def get_metric_json(metric): def get_metric_json(metric):
@ -458,15 +479,17 @@ def get_metric_json(metric):
@app.route('/api/1.0/project_types') @app.route('/api/1.0/project_types')
@decorators.jsonify('project_types') @decorators.response()
@decorators.exception_handler() @decorators.exception_handler()
@decorators.query_filter(query_param='query') @decorators.cached(ignore=parameters.FILTER_PARAMETERS)
def get_project_types_json(query_filter): @decorators.jsonify('project_types')
def get_project_types_json(**kwargs):
return [{'id': pt['id'], 'text': pt['title'], 'items': pt.get('items', [])} return [{'id': pt['id'], 'text': pt['title'], 'items': pt.get('items', [])}
for pt in vault.get_project_types() if query_filter(pt['title'])] for pt in vault.get_project_types()]
@app.route('/api/1.0/project_types/<project_type>') @app.route('/api/1.0/project_types/<project_type>')
@decorators.response()
@decorators.jsonify('project_type') @decorators.jsonify('project_type')
@decorators.exception_handler() @decorators.exception_handler()
def get_project_type_json(project_type): def get_project_type_json(project_type):
@ -487,8 +510,10 @@ def _get_week(kwargs, param_name):
@app.route('/api/1.0/stats/timeline') @app.route('/api/1.0/stats/timeline')
@decorators.jsonify('timeline')
@decorators.exception_handler() @decorators.exception_handler()
@decorators.response()
@decorators.cached()
@decorators.jsonify('timeline')
@decorators.record_filter(ignore=['release', 'start_date']) @decorators.record_filter(ignore=['release', 'start_date'])
def timeline(records, **kwargs): def timeline(records, **kwargs):
# find start and end dates # find start and end dates

View File

@ -42,7 +42,7 @@ def date_to_timestamp(d):
def date_to_timestamp_ext(d): def date_to_timestamp_ext(d):
try: try:
return date_to_timestamp(d) return date_to_timestamp(d)
except ValueError: except (ValueError, TypeError):
return int(d) return int(d)
@ -72,6 +72,10 @@ def timestamp_to_day(timestamp):
return timestamp // (24 * 3600) return timestamp // (24 * 3600)
def round_timestamp_to_day(timestamp):
return (int(timestamp) // (24 * 3600)) * (24 * 3600)
def check_email_validity(email): def check_email_validity(email):
if email: if email:
return re.match(r'[\w\d_\.-]+@([\w\d_\.-]+\.)+[\w]+', email) return re.match(r'[\w\d_\.-]+@([\w\d_\.-]+\.)+[\w]+', email)

View File

@ -70,11 +70,6 @@ class TestAPICompanies(test_api.TestAPI):
self.assertEqual([{'id': 'ibm', 'text': 'IBM'}, self.assertEqual([{'id': 'ibm', 'text': 'IBM'},
{'id': 'nec', 'text': 'NEC'}], companies) {'id': 'nec', 'text': 'NEC'}], companies)
response = self.app.get('/api/1.0/companies?metric=commits&'
'company_name=ib&module=glance')
companies = json.loads(response.data)['companies']
self.assertEqual([{'id': 'ibm', 'text': 'IBM'}], companies)
def test_get_company(self): def test_get_company(self):
with test_api.make_runtime_storage( with test_api.make_runtime_storage(
{'repos': [{'module': 'nova', 'project_type': 'openstack', {'repos': [{'module': 'nova', 'project_type': 'openstack',

View File

@ -68,14 +68,6 @@ class TestAPIModules(test_api.TestAPI):
'module groups that are completely within ' 'module groups that are completely within '
'project type') 'project type')
response = self.app.get('/api/1.0/modules?query=glance&'
'project_type=all&metric=commits')
modules = json.loads(response.data)['modules']
self.assertEqual(
[{'id': 'glance', 'text': 'glance', 'tag': 'module'}],
modules,
message='Expected modules which name contains query')
def test_get_module(self): def test_get_module(self):
with test_api.make_runtime_storage( with test_api.make_runtime_storage(
{'repos': [{'module': 'nova', 'organization': 'openstack', {'repos': [{'module': 'nova', 'organization': 'openstack',

View File

@ -20,39 +20,25 @@ from tests.api import test_api
class TestAPIReleases(test_api.TestAPI): class TestAPIReleases(test_api.TestAPI):
def test_releases_empty(self):
with test_api.make_runtime_storage({}):
response = self.app.get('/api/1.0/releases')
self.assertEqual(200, response.status_code)
def test_releases(self): def test_releases(self):
with test_api.make_runtime_storage( with test_api.make_runtime_storage(
{'releases': [ {'releases': [
{'release_name': 'prehistory', 'end_date': 1365033600}, {'release_name': 'prehistory', 'end_date': 1365033600},
{'release_name': 'havana', 'end_date': 1381968000}, {'release_name': 'havana', 'end_date': 1381968000},
{'release_name': 'icehouse', 'end_date': 1397692800}]}): {'release_name': 'icehouse', 'end_date': 1397692800}]},
test_api.make_records(record_type=['commit'])):
response = self.app.get('/api/1.0/releases') response = self.app.get('/api/1.0/releases')
releases = json.loads(response.data)['releases'] releases = json.loads(response.data)['releases']
self.assertEqual(3, len(releases)) self.assertEqual(3, len(releases))
self.assertIn({'id': 'all', 'text': 'All'}, releases) self.assertIn({'id': 'all', 'text': 'All'}, releases)
self.assertIn({'id': 'icehouse', 'text': 'Icehouse'}, releases) self.assertIn({'id': 'icehouse', 'text': 'Icehouse'}, releases)
def test_releases_search(self):
with test_api.make_runtime_storage(
{'releases': [
{'release_name': 'prehistory', 'end_date': 1365033600},
{'release_name': 'havana', 'end_date': 1381968000},
{'release_name': 'icehouse', 'end_date': 1397692800}]}):
response = self.app.get('/api/1.0/releases?query=hav')
releases = json.loads(response.data)['releases']
self.assertEqual(1, len(releases))
self.assertIn({'id': 'havana', 'text': 'Havana'}, releases)
def test_release_details(self): def test_release_details(self):
with test_api.make_runtime_storage( with test_api.make_runtime_storage(
{'releases': [ {'releases': [
{'release_name': 'prehistory', 'end_date': 1365033600}, {'release_name': 'prehistory', 'end_date': 1365033600},
{'release_name': 'icehouse', 'end_date': 1397692800}]}): {'release_name': 'icehouse', 'end_date': 1397692800}]},
test_api.make_records(record_type=['commit'])):
response = self.app.get('/api/1.0/releases/icehouse') response = self.app.get('/api/1.0/releases/icehouse')
release = json.loads(response.data)['release'] release = json.loads(response.data)['release']
self.assertEqual({'id': 'icehouse', 'text': 'Icehouse'}, release) self.assertEqual({'id': 'icehouse', 'text': 'Icehouse'}, release)

View File

@ -41,26 +41,6 @@ class TestAPIUsers(test_api.TestAPI):
self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users) self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users)
self.assertIn({'id': 'bill_smith', 'text': 'Bill Smith'}, users) self.assertIn({'id': 'bill_smith', 'text': 'Bill Smith'}, users)
def test_users_search(self):
with test_api.make_runtime_storage(
{'repos': [{'module': 'nova', 'organization': 'openstack',
'uri': 'git://github.com/openstack/nova.git'}],
'project_types': [
{'id': 'openstack', 'title': 'openstack',
'modules': ['nova', 'glance']}],
'module_groups': {
'nova': test_api.make_module('nova'),
'glance': test_api.make_module('glance')},
'user:john_doe': {'user_name': 'John Doe'},
'user:bill_smith': {'user_name': 'Bill Smith'}},
test_api.make_records(record_type=['commit'], module=['nova'],
user_name=['John Doe', 'Bill Smith'])):
response = self.app.get('/api/1.0/users?'
'module=nova&query=doe&metric=commits')
users = json.loads(response.data)['users']
self.assertEqual(1, len(users))
self.assertIn({'id': 'john_doe', 'text': 'John Doe'}, users)
def test_user_details(self): def test_user_details(self):
with test_api.make_runtime_storage( with test_api.make_runtime_storage(
{'user:john_doe': { {'user:john_doe': {