Activity log is re-implemented for dynamic load

* Record filter to return records referred by blueprints
* Common code for activity stream processing is extracted into template

Change-Id: I1d36ba2be3c76fafcf25a09ffbb40cfc083271da

dashboard/decorators.py

@@ -88,9 +88,11 @@ def record_filter(ignore=None, use_default=True):
 
             if 'metric' not in ignore:
                 metrics = parameters.get_parameter(kwargs, 'metric')
+                if 'all' not in metrics:
                     for metric in metrics:
-                    record_ids &= memory_storage_inst.get_record_ids_by_type(
+                        record_ids &= (
-                        parameters.METRIC_TO_RECORD_TYPE[metric])
+                            memory_storage_inst.get_record_ids_by_type(
+                                parameters.METRIC_TO_RECORD_TYPE[metric]))
 
                 if 'tm_marks' in metrics:
                     filtered_ids = []
@@ -104,6 +106,13 @@ def record_filter(ignore=None, use_default=True):
                             filtered_ids.append(record['record_id'])
                     record_ids = filtered_ids
 
+            if 'blueprint_id' not in ignore:
+                param = parameters.get_parameter(kwargs, 'blueprint_id')
+                if param:
+                    record_ids &= (
+                        memory_storage_inst.get_record_ids_by_blueprint_ids(
+                            param))
+
             kwargs['records'] = memory_storage_inst.get_records(record_ids)
             return f(*args, **kwargs)
 
@@ -167,7 +176,7 @@ def aggregate_filter():
                 'bpc': (incremental_filter, None),
             }
             if metric not in metric_to_filters_map:
-                raise Exception('Invalid metric %s' % metric)
+                metric = parameters.get_default('metric')
 
             kwargs['metric_filter'] = metric_to_filters_map[metric][0]
             kwargs['finalize_handler'] = metric_to_filters_map[metric][1]
@@ -241,6 +250,9 @@ def templated(template=None, return_code=200):
                                            key=lambda x: x[0])
 
             ctx['company'] = parameters.get_single_parameter(kwargs, 'company')
+            ctx['company_original'] = (
+                vault.get_memory_storage().get_original_company_name(
+                    ctx['company']))
             ctx['module'] = parameters.get_single_parameter(kwargs, 'module')
             ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
             ctx['page_title'] = helpers.make_page_title(

dashboard/reports.py

@@ -21,7 +21,7 @@ import time
 
 import flask
 
-from dashboard import decorators, parameters
+from dashboard import decorators
 from dashboard import helpers
 from dashboard import vault
 from stackalytics.processor import utils
@@ -34,7 +34,7 @@ blueprint = flask.Blueprint('reports', __name__, url_prefix='/report')
 @decorators.templated()
 @decorators.exception_handler()
 def blueprint_summary(module, blueprint_name):
-    blueprint_id = module + ':' + blueprint_name
+    blueprint_id = utils.get_blueprint_id(module, blueprint_name)
     bpd = vault.get_memory_storage().get_record_by_primary_key(
         'bpd:' + blueprint_id)
     if not bpd:
@@ -136,7 +136,7 @@ def _get_punch_card_data(records):
     if punch_card_raw[6][23] == 0:
         punch_card_data.append([23, 6, 0, 0])
 
-    return punch_card_data
+    return json.dumps(punch_card_data)
 
 
 @blueprint.route('/users/<user_id>')
@@ -151,17 +151,13 @@ def user_activity(user_id):
     memory_storage_inst = vault.get_memory_storage()
     records = memory_storage_inst.get_records(
         memory_storage_inst.get_record_ids_by_user_ids([user_id]))
-
+    records = sorted(records, key=operator.itemgetter('date'), reverse=True)
-    activity = helpers.get_activity(records, 0, -1)
-
-    punch_card_data = _get_punch_card_data(activity)
 
     return {
         'user': user,
-        'activity': activity[:parameters.DEFAULT_STATIC_ACTIVITY_SIZE],
+        'total_records': len(records),
-        'total_records': len(activity),
+        'contribution': helpers.get_contribution_summary(records),
-        'contribution': helpers.get_contribution_summary(activity),
+        'punch_card_data': _get_punch_card_data(records),
-        'punch_card_data': json.dumps(punch_card_data),
     }
 
 
@@ -175,17 +171,13 @@ def company_activity(company):
     memory_storage_inst = vault.get_memory_storage()
     records = memory_storage_inst.get_records(
         memory_storage_inst.get_record_ids_by_companies([original_name]))
-
+    records = sorted(records, key=operator.itemgetter('date'), reverse=True)
-    activity = helpers.get_activity(records, 0, -1)
-
-    punch_card_data = _get_punch_card_data(activity)
 
     return {
-        'company': original_name,
+        'company_name': original_name,
-        'activity': activity[:parameters.DEFAULT_STATIC_ACTIVITY_SIZE],
+        'total_records': len(records),
-        'total_records': len(activity),
+        'contribution': helpers.get_contribution_summary(records),
-        'contribution': helpers.get_contribution_summary(activity),
+        'punch_card_data': _get_punch_card_data(records),
-        'punch_card_data': json.dumps(punch_card_data),
     }
 
 

dashboard/templates/_macros/activity_log.html

@@ -0,0 +1,131 @@
+{% macro show_activity_log(user_id=None, company=None, blueprint_id=None, show_gravatar=True, gravatar_size=32) -%}
+
+<script type="text/javascript">
+    var page_size = 10;
+    var start_record = 0;
+    var uri_options = {project_type: "all", release: "all", metric: "all"};
+    {% if user_id %}
+        uri_options["user_id"] = "{{ user_id }}";
+    {% endif %}
+    {% if company %}
+        uri_options["company"] = "{{ company }}";
+    {% endif %}
+    {% if blueprint_id %}
+        uri_options["blueprint_id"] = "{{ blueprint_id }}";
+    {% endif %}
+
+    function load_activity(extra_options) {
+        var options = {page_size: page_size, start_record: start_record};
+        $.extend(options, extra_options);
+
+        $.ajax({
+            url: make_uri("/api/1.0/activity", options),
+            dataType: "json",
+            success: function (data) {
+                if (data["activity"].length < page_size) {
+                    $('#activity_more').hide();
+                }
+                if ((start_record == 0) && (data["activity"].length == 0)) {
+                    $('#activity_header').hide();
+                }
+                $("#activity_template").tmpl(data["activity"]).appendTo("#activity_container");
+                $('.ext_link').click(function (event) {
+                    event.preventDefault();
+                    event.stopPropagation();
+                    window.open(this.href, '_blank');
+                });
+                $(".expand-button").click(function () {
+                    $("#content-" + this.id).slideToggle('fast');
+                });
+            }
+        });
+    }
+    $(document).ready(function () {
+        load_activity(uri_options);
+    });
+    $(document).ready(function () {
+        $('#activity_more')
+                .click(function () {
+                    start_record += page_size;
+                    load_activity(uri_options)
+                });
+    });
+</script>
+
+<script id="activity_template" type="text/x-jquery-tmpl">
+    <div style="margin-bottom: 1em;">
+    <div style='float: left; '>
+        <img src="${gravatar}" style="width: {{ gravatar_size }}px; height: {{ gravatar_size }}px;">
+    </div>
+
+    <div style="margin-left: {{ gravatar_size * 1.4 }}px;">
+
+    {% raw %}
+        <div style="font-weight: bold;">{%html author_link %} ({%html company_link %})</div>
+        <div style="font-weight: bold;">${date_str} in {%html module_link%}</div>
+
+        {%if record_type == "commit" %}
+            {%if correction_comment != "" %}
+            <div style='font-weight: bold; color: red;'>Commit corrected:
+                <span>${correction_comment}</span></div>
+            {%/if%}
+            <div style='font-weight: bold;'>${subject}</div>
+            <div style='white-space: pre-wrap; '>{%html message %}</div>
+            <div><span style="color: green">+<span>${lines_added}</span></span>
+                <span style="color: blue">- <span>${lines_deleted}</span></span></div>
+        {%elif record_type == "mark" %}
+            <div>Review #${review_number} submitted by {%html parent_author_link %}</div>
+            <div style='font-weight: bold;'>${subject}</div>
+            <div>Change Id: <a href="${url}">${review_id}</a></div>
+            <div style="color: {%if value > 0 %} green {%else%} blue {%/if%}">${description}: <span class="review_mark">${value}</span></div>
+        {%elif record_type == "review" %}
+            <div style='font-weight: bold;'>${subject}</div>
+            <div>Change Id: <a href="${url}">${id}</a></div>
+        {%elif record_type == "email" %}
+            <div style='font-weight: bold;'>
+            {%if email_link != "" %}
+            <a href='${email_link}'>
+            {%/if%}
+            ${subject}
+            {%if email_link != "" %}
+            </a>
+            {%/if%}
+            </div>
+            {%if blueprint_id_count %}
+            <div>Mentions blueprints:
+            {%each( index, value ) blueprint_id %}
+                ${value}
+            {%/each%}
+            </div>
+            {%/if%}
+            {%if body %}
+            <div>Email: <span class="expand-button" id="button-${record_id}">[+]</span></div>
+            <div id="content-button-${record_id}" class="message" style="display:none;">${body}</div>
+            {%/if%}
+        {%elif ((record_type == "bpd") || (record_type == "bpc")) %}
+            <div style='font-weight: bold;'>${title} ({%html blueprint_link %})</div>
+            <div style='white-space: pre-wrap;'>${summary}</div>
+
+            <div>Priority: <span class="specpriority${priority}">${priority}</span></div>
+            <div>Status: <span class="status${lifecycle_status}">${lifecycle_status}</span>
+                (<span class="specstatus${definition_status}">${definition_status}</span>,
+                <span class="specdelivery${implementation_status}">${implementation_status}</span>)</div>
+            {%if mention_count %}
+            <div><b>Mention count: ${mention_count}, last mention on ${mention_date_str}</b></div>
+            {%/if%}
+        {%/if%}
+    </div>
+    </div>
+    {% endraw %}
+</script>
+
+    <h2 id="activity_header">Activity Log</h2>
+    <div id="activity_container"></div>
+
+    <div style="height: 44px;">
+        <div class="dataTables_paginate paging_full_numbers" id="activity_paginate">
+            <a class="last paginate_button" tabindex="0" id="activity_more">More...</a>
+        </div>
+    </div>
+
+{%- endmacro %}

dashboard/templates/overview.html

@@ -11,7 +11,6 @@
 {% set show_module_contribution = (module) and (not user_id) %}
 {% set show_contribution = (show_user_contribution) or (show_module_contribution) %}
 {% set show_user_profile = (user_id) %}
-{% set show_module_profile = (module) and (not user_id) and (not company) %}
 {% set show_top_mentors_options = (metric == 'tm_marks') %}
 {% set show_review_ratio = (metric in ['marks', 'tm_marks']) %}
 
@@ -187,6 +186,15 @@
 <div>Draft Blueprints: <b>${drafted_blueprint_count}</b></div>
 <div>Completed Blueprints: <b>${completed_blueprint_count}</b></div>
 <div>Emails: <b>${email_count}</b></div>
+{% if module %}
+<div><b><a href="/report/reviews/{{ module }}" target="_blank">Show open reviews for {{ module }}↗</a></b></div>
+{% endif %}
+{% if company %}
+<div><b><a href="/report/companies/{{ company }}" target="_blank">Show activity report for {{ company_original }}↗</a></b></div>
+{% endif %}
+{% if user_id %}
+<div><b><a href="/report/users/{{ user_id }}" target="_blank">Show activity report for {{ user_id }}↗</a></b></div>
+{% endif %}
 </script>
 
 {% endblock %}
@@ -291,6 +299,10 @@
         </div>
     {% endif %}
 
+    {% if show_module_contribution %}
+        <div id="contribution_container"></div>
+    {% endif %}
+
 {% endblock %}
 
 {% block right_frame %}
@@ -316,13 +328,6 @@
     </div>
     {% endif %}
 
-    {% if show_module_profile %}
-        <h2>Module {{ module }}</h2>
-        <div><a href="/report/reviews/{{ module }}" target="_blank">Open reviews report↗</a></div>
-
-        <div id="contribution_container"></div>
-    {% endif %}
-
     {% if show_bp_breakdown %}
     <div id="bp_container">
     <h2>Blueprint popularity</h2>

dashboard/templates/reports/base_report.html

@@ -1,7 +1,10 @@
-<!DOCTYPE html>
+{% extends "base.html" %}
-<html>
+
-<head>
+{% block head %}
     <title>{% block title %}{% endblock %}</title>
+
+    <meta name="keywords" content="openstack, contribution, statistics, community, review, commit, report, havana, grizzly, icehouse"/>
+
     <link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/style.css') }}">
 
     <link href='http://fonts.googleapis.com/css?family=PT+Sans:400,700,400italic&subset=latin,cyrillic' rel='stylesheet' type='text/css' />
@@ -34,10 +37,8 @@
     <script type="text/javascript" src="{{ url_for('static', filename='js/jquery.tmpl.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/stackalytics-ui.js') }}"></script>
 
-{% block head %}{% endblock %}
+    {% block scripts %}{% endblock %}
-
+{% endblock %}
-</head>
-<body style="margin: 2em;">
 
 {% macro show_activity_log(activity, show_gravatar) -%}
 
@@ -143,12 +144,16 @@
 
 {%- endmacro %}
 
+{% block body %}
+
+<div style="margin: 2em;">
 <div id="analytics_header" style="padding-bottom: 1em; border-bottom: 1px solid darkgrey;">
     <span id="logo"><a href="/">Stackalytics</a></span>
     <span id="slogan">| community heartbeat</span>
 </div>
 
-{% block body %}{% endblock %}
+{% block content %}
+{% endblock %}
 
-</body>
+</div>
-</html>
+{% endblock %}

dashboard/templates/reports/blueprint_summary.html

@@ -1,10 +1,11 @@
 {% extends "reports/base_report.html" %}
+{% import '_macros/activity_log.html' as activity_log %}
 
 {% block title %}
 Blueprint “{{ blueprint.title }}” report
 {% endblock %}
 
-{% block body %}
+{% block content %}
 <h1>Blueprint &ldquo;{{ blueprint.name }}&rdquo;</h1>
 
 <div><span class="label">Title:</span> {{ blueprint.title }}</div>
@@ -31,5 +32,6 @@ Blueprint &ldquo;{{ blueprint.title }}&rdquo; report
 <div class="message">{{  blueprint.whiteboard }}</div>
 {% endif %}
 
-{{ show_activity_log(activity, true) }}
+{{ activity_log.show_activity_log(blueprint_id=blueprint.id, show_gravatar=false, gravatar_size=64) }}
+
 {% endblock %}

dashboard/templates/reports/company_activity.html

@@ -1,10 +1,11 @@
 {% extends "reports/base_report.html" %}
+{% import '_macros/activity_log.html' as activity_log %}
 
 {% block title %}
-{{ company }} activity report
+{{ company_name }} activity in OpenStack
 {% endblock %}
 
-{% block head %}
+{% block scripts %}
     <script type="text/javascript">
         $(document).ready(function () {
             render_punch_card("punch_card", [{{ punch_card_data }}]);
@@ -12,13 +13,13 @@
     </script>
 {% endblock %}
 
-{% block body %}
+{% block content %}
-<h1>{{ company }} activity report</h1>
+<h1>{{ company_name }} activity report</h1>
 
 {{ show_contribution_summary(contribution) }}
 
 <div id="punch_card" style="width: 100%; height: 350px;"></div>
 
-{{ show_activity_log(activity, false) }}
+{{ activity_log.show_activity_log(company=company_name, show_gravatar=false, gravatar_size=64) }}
 
 {% endblock %}

dashboard/templates/reports/open_reviews.html

@@ -4,7 +4,7 @@
 Open reviews report for {{ module }}
 {% endblock %}
 
-{% block head %}
+{% block scripts %}
     <script type="text/javascript">
         $(document).ready(function () {
             render_bar_chart("latest_revision_chart", [{{ latest_revision.chart_data }}]);
@@ -13,7 +13,7 @@ Open reviews report for {{ module }}
     </script>
 {% endblock %}
 
-{% block body %}
+{% block content %}
 <h1>Open reviews for {{ module }}</h1>
 
 <h3>Summary</h3>

dashboard/templates/reports/user_activity.html

@@ -1,10 +1,11 @@
 {% extends "reports/base_report.html" %}
+{% import '_macros/activity_log.html' as activity_log %}
 
 {% block title %}
-{{ user.user_name }} activity report
+{{ user.user_name }} activity in OpenStack
 {% endblock %}
 
-{% block head %}
+{% block scripts %}
     <script type="text/javascript">
         $(document).ready(function () {
             render_punch_card("punch_card", [{{ punch_card_data }}]);
@@ -12,13 +13,13 @@
     </script>
 {% endblock %}
 
-{% block body %}
+{% block content %}
 <h1>{{ user.user_name }} activity report</h1>
 
 {{ show_contribution_summary(contribution) }}
 
 <div id="punch_card" style="width: 100%; height: 350px;"></div>
 
-{{ show_activity_log(activity, false) }}
+{{ activity_log.show_activity_log(user_id=user.user_id, show_gravatar=false, gravatar_size=64) }}
 
 {% endblock %}

dashboard/web.py

@@ -142,14 +142,16 @@ def get_activity_json(records):
     start_record = int(flask.request.args.get('start_record') or 0)
     page_size = int(flask.request.args.get('page_size') or
                     parameters.DEFAULT_RECORDS_LIMIT)
+    records_sorted = sorted(records, key=lambda x: x['date'], reverse=True)
+    records_sorted = records_sorted[start_record:start_record + page_size]
+
     result = []
-    for record in records:
+    for record in records_sorted:
         processed_record = helpers.extend_record(record)
         if processed_record:
             result.append(processed_record)
 
-    result.sort(key=lambda x: x['date'], reverse=True)
+    return result
-    return result[start_record:start_record + page_size]
 
 
 @app.route('/api/1.0/contribution')

stackalytics/processor/lp.py

@@ -49,7 +49,7 @@ def log(repo):
                 record[field] = utils.iso8601_to_timestamp(date)
 
         record['module'] = module
-        record['id'] = module + ':' + record['name']
+        record['id'] = utils.get_blueprint_id(module, record['name'])
 
         LOG.debug('New blueprint: %s', record)
         yield record

stackalytics/processor/utils.py

@@ -133,3 +133,7 @@ def merge_records(original, new):
             need_update = True
             original[key] = value
     return need_update
+
+
+def get_blueprint_id(module, name):
+    return module + ':' + name