Implemented user activity report and punch card

* Added new report that shows activity for a given user
* Refactored code that calculates activity log and contribution summary
* Implemented punch card that shows when user is active mostly

Implements blueprint review-punchcard

Change-Id: I17af4edb65e49c934c503c5ab6e5d2b6212780b0

dashboard/helpers.py

@@ -44,44 +44,106 @@ def _extend_record_common_fields(record):
 
 
 def extend_record(record):
+    record = record.copy()
+    _extend_record_common_fields(record)
+
     if record['record_type'] == 'commit':
-        commit = record.copy()
+        record['branches'] = ','.join(record['branches'])
-        commit['branches'] = ','.join(commit['branches'])
+        if 'correction_comment' not in record:
-        if 'correction_comment' not in commit:
+            record['correction_comment'] = ''
-            commit['correction_comment'] = ''
+        record['message'] = make_commit_message(record)
-        commit['message'] = make_commit_message(record)
-        _extend_record_common_fields(commit)
-        return commit
     elif record['record_type'] == 'mark':
-        review = record.copy()
         parent = vault.get_memory_storage().get_record_by_primary_key(
-            review['review_id'])
+            record['review_id'])
-        if parent:
+        if not parent:
-            review['review_number'] = parent.get('review_number')
+            return None
-            review['subject'] = parent['subject']
+
-            review['url'] = parent['url']
+        for k, v in parent.iteritems():
-            review['parent_author_link'] = make_link(
+            record['parent_%s' % k] = v
+        record['review_number'] = parent.get('review_number')
+        record['subject'] = parent['subject']
+        record['url'] = parent['url']
+        record['parent_author_link'] = make_link(
             parent['author_name'], '/',
-                {'user_id': parent['user_id'],
+            {'user_id': parent['user_id'], 'company': ''})
-                 'company': ''})
-            _extend_record_common_fields(review)
-            return review
     elif record['record_type'] == 'email':
-        email = record.copy()
+        record['email_link'] = record.get('email_link') or ''
-        _extend_record_common_fields(email)
+    elif record['record_type'] in ['bpd', 'bpc']:
-        email['email_link'] = email.get('email_link') or ''
+        record['summary'] = utils.format_text(record['summary'])
-        return email
-    elif ((record['record_type'] == 'bpd') or
-          (record['record_type'] == 'bpc')):
-        blueprint = record.copy()
-        _extend_record_common_fields(blueprint)
-        blueprint['summary'] = utils.format_text(record['summary'])
         if record.get('mention_count'):
-            blueprint['mention_date_str'] = format_datetime(
+            record['mention_date_str'] = format_datetime(
                 record['mention_date'])
-        blueprint['blueprint_link'] = make_blueprint_link(
+        record['blueprint_link'] = make_blueprint_link(
-            blueprint['name'], blueprint['module'])
+            record['name'], record['module'])
-        return blueprint
+
+    return record
+
+
+def extend_user(user):
+    user = user.copy()
+
+    user['id'] = user['user_id']
+    user['text'] = user['user_name']
+    if user['companies']:
+        company_name = user['companies'][-1]['company_name']
+        user['company_link'] = make_link(
+            company_name, '/', {'company': company_name, 'user_id': ''})
+    else:
+        user['company_link'] = ''
+    if user['emails']:
+        user['gravatar'] = gravatar(user['emails'][0])
+    else:
+        user['gravatar'] = gravatar(user['user_id'])
+
+    return user
+
+
+def get_activity(records, start_record=0,
+                 page_size=parameters.DEFAULT_RECORDS_LIMIT):
+    result = []
+    for record in records:
+        processed_record = extend_record(record)
+        if processed_record:
+            result.append(processed_record)
+
+    result.sort(key=lambda x: x['date'], reverse=True)
+    if page_size == -1:
+        return result[start_record:]
+    else:
+        return result[start_record:start_record + page_size]
+
+
+def get_contribution_summary(records):
+    marks = dict((m, 0) for m in [-2, -1, 0, 1, 2])
+    commit_count = 0
+    loc = 0
+    drafted_blueprint_count = 0
+    completed_blueprint_count = 0
+    email_count = 0
+
+    for record in records:
+        record_type = record['record_type']
+        if record_type == 'commit':
+            commit_count += 1
+            loc += record['loc']
+        elif record['record_type'] == 'mark':
+            marks[record['value']] += 1
+        elif record['record_type'] == 'email':
+            email_count += 1
+        elif record['record_type'] == 'bpd':
+            drafted_blueprint_count += 1
+        elif record['record_type'] == 'bpc':
+            completed_blueprint_count += 1
+
+    result = {
+        'drafted_blueprint_count': drafted_blueprint_count,
+        'completed_blueprint_count': completed_blueprint_count,
+        'commit_count': commit_count,
+        'email_count': email_count,
+        'loc': loc,
+        'marks': marks,
+    }
+    return result
 
 
 def format_datetime(timestamp):

dashboard/parameters.py

@@ -48,6 +48,7 @@ METRIC_TO_RECORD_TYPE = {
 }
 
 DEFAULT_RECORDS_LIMIT = 10
+DEFAULT_STATIC_ACTIVITY_SIZE = 50
 
 
 def get_default(param_name):

dashboard/reports.py

@@ -12,14 +12,16 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import json
 
+import datetime
 import operator
 import time
 
 import flask
 
-from dashboard import decorators
+from dashboard import decorators, parameters
 from dashboard import helpers
 from dashboard import vault
 from stackalytics.processor import utils
@@ -113,6 +115,50 @@ def open_reviews(module):
     }
 
 
+def _get_punch_card_data(records):
+    punch_card_raw = []  # matrix days x hours
+    for wday in xrange(0, 7):
+        punch_card_raw.append([0] * 24)
+    for record in records:
+        tt = datetime.datetime.fromtimestamp(record['date']).timetuple()
+        punch_card_raw[tt.tm_wday][tt.tm_hour] += 1
+
+    punch_card_data = []  # format for jqplot bubble renderer
+    for wday in xrange(0, 7):
+        for hour in xrange(0, 24):
+            v = punch_card_raw[wday][hour]
+            if v:
+                punch_card_data.append([hour, wday, v, v])
+
+    return punch_card_data
+
+
+@blueprint.route('/users/<user_id>')
+@decorators.templated()
+@decorators.exception_handler()
+def user_activity(user_id):
+    user = vault.get_user_from_runtime_storage(user_id)
+    if not user:
+        flask.abort(404)
+    user = helpers.extend_user(user)
+
+    memory_storage_inst = vault.get_memory_storage()
+    records = memory_storage_inst.get_records(
+        memory_storage_inst.get_record_ids_by_user_ids([user_id]))
+
+    activity = helpers.get_activity(records, 0, -1)
+
+    punch_card_data = _get_punch_card_data(activity)
+
+    return {
+        'user': user,
+        'activity': activity[:parameters.DEFAULT_STATIC_ACTIVITY_SIZE],
+        'total_records': len(activity),
+        'contribution': helpers.get_contribution_summary(activity),
+        'punch_card_data': json.dumps(punch_card_data),
+    }
+
+
 @blueprint.route('/large_commits')
 @decorators.jsonify('commits')
 @decorators.exception_handler()

dashboard/static/js/stackalytics-ui.js

@@ -196,7 +196,50 @@ function render_bar_chart(chart_id, chart_data) {
                 label: "Age"
             },
             yaxis: {
-                label: "Count"
+                label: "Count",
+                labelRenderer: $.jqplot.CanvasAxisLabelRenderer
+            }
+        }
+    });
+}
+
+function render_punch_card(chart_id, chart_data) {
+    $.jqplot(chart_id, chart_data, {
+        seriesDefaults:{
+            renderer: $.jqplot.BubbleRenderer,
+            rendererOptions: {
+                varyBubbleColors: false,
+                color: '#a09898',
+                autoscalePointsFactor: -0.25,
+                highlightAlpha: 0.7
+            },
+            shadow: true,
+            shadowAlpha: 0.05
+        },
+        axesDefaults: {
+            tickRenderer: $.jqplot.CanvasAxisTickRenderer
+        },
+        axes: {
+            xaxis: {
+                label: 'Hour',
+                labelRenderer: $.jqplot.CanvasAxisLabelRenderer,
+                tickOptions: {
+                    formatter: function (format, val) {
+                        if (val < 0 || val > 24) { return "" }
+                        return val;
+                    }
+                }
+            },
+            yaxis: {
+                label: 'Day of week',
+                labelRenderer: $.jqplot.CanvasAxisLabelRenderer,
+                tickOptions: {
+                    formatter: function (format, val) {
+                        if (val < 0 || val > 6) { return "" }
+                        var labels = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"];
+                        return labels[val];
+                    }
+                }
             }
         }
     });

dashboard/templates/overview.html

@@ -184,8 +184,8 @@
 <div>Total LOC: <b>${loc}</b></div>
 <div>Review stat (-2, -1, +1, +2): <b>${marks["-2"]}, ${marks["-1"]}, ${marks["1"]}, ${marks["2"]}</b></div>
     {% endraw %}
-<div>Draft Blueprints: <b>${new_blueprint_count}</b></div>
+<div>Draft Blueprints: <b>${drafted_blueprint_count}</b></div>
-<div>Completed Blueprints: <b>${competed_blueprint_count}</b></div>
+<div>Completed Blueprints: <b>${completed_blueprint_count}</b></div>
 <div>Emails: <b>${email_count}</b></div>
 </script>
 

dashboard/templates/reports/base_report.html

@@ -22,9 +22,11 @@
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.json2.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.pieRenderer.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.barRenderer.min.js') }}"></script>
+    <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.bubbleRenderer.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.categoryAxisRenderer.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.dateAxisRenderer.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.canvasTextRenderer.min.js') }}"></script>
+    <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.canvasAxisLabelRenderer.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.canvasAxisTickRenderer.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.cursor.min.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.highlighter.min.js') }}"></script>
@@ -37,22 +39,29 @@
 </head>
 <body style="margin: 2em;">
 
-{% macro show_activity_log(activity) -%}
+{% macro show_activity_log(activity, show_gravatar) -%}
 
 <h2>Activity Log</h2>
 
 {% if not activity %}
     <div>No activity.</div>
-{% endif %}
+{% else %}
+
 {% for item in activity %}
     <div style="margin-bottom: 1em;">
-        <div style='float: left; '><img src="{{ item.author_email | gravatar(size=64) }}">
+        <div style='float: left; '>
+            {% if show_gravatar %}
+            <img src="{{ item.gravatar }}">
+            {% else %}
+            <img src="{{ item.record_type | gravatar(default='retro') }}">
+            {% endif %}
         </div>
+
         <div style="margin-left: 80px;">
             <div style="font-weight: bold;">{{ item.date_str}}</div>
             <div style="font-weight: bold;">{{ item.author_name }} ({{ item.company_name }})</div>
             {% if item.record_type == "commit" %}
-                <div style='font-weight: bold;'>Commit &ldquo;{{ item.subject }}&rdquo;</div>
+                <div style='font-weight: bold;'>Commit &ldquo;{{ item.subject }}&rdquo; to {{ item.module }}</div>
                 <div class="message">{{ item.message | safe }}</div>
                 <div><span style="color: green">+<span>{{ item.lines_added }}</span></span>
                     <span style="color: blue">- <span>{{ item.lines_deleted }}</span></span>
@@ -62,11 +71,15 @@
                         <span>{{ item.correction_comment }}</span></div>
                 {% endif %}
             {% elif item.record_type == "mark" %}
-                <div style='font-weight: bold;'>Review &ldquo;{{item.subject}}&rdquo;</div>
+                <div style='font-weight: bold;'>Review &ldquo;{{item.subject}}&rdquo; in {{ item.module }}</div>
-                <div>Patch submitted by {{ parent_author_link }}</div>
+                <div>Patch submitted by <a href="{{ item.parent_user_id }}">{{ item.parent_author_name }}</a></div>
                 <div>Change Id: <a href="{{item.url}}">{{item.review_id}}</a></div>
                 <div style="color: {% if item.value > 0 %} green {% else %} blue {% endif %}">
                     {{item.description}}: <span class="review_mark">{{item.value}}</span></div>
+            {% elif item.record_type == "review" %}
+                <div style='font-weight: bold;'>Patch &ldquo;{{item.subject}}&rdquo; in {{ item.module }}</div>
+                <div>URL: <a href="{{item.url}}">{{item.url}}</a></div>
+                <div>Change Id: <a href="{{item.url}}">{{item.id}}</a></div>
             {% elif item.record_type == "email" %}
                 <div style='font-weight: bold;'>Email &ldquo;{{item.subject}}&rdquo;</div>
                 {% if item.body %}
@@ -77,6 +90,7 @@
     </div>
 
 {% endfor %}
+{% endif %}
 
 {%- endmacro %}
 

dashboard/templates/reports/blueprint_summary.html

@@ -31,6 +31,5 @@ Blueprint {{ blueprint.title }} detailed report
 <div class="message">{{  blueprint.whiteboard }}</div>
 {% endif %}
 
-
+{{ show_activity_log(activity, true) }}
-{{ show_activity_log(activity) }}
 {% endblock %}

dashboard/templates/reports/user_activity.html

@@ -0,0 +1,35 @@
+{% extends "reports/base_report.html" %}
+
+{% block title %}
+{{ user.user_name }} activity report
+{% endblock %}
+
+{% block head %}
+    <script type="text/javascript">
+        $(document).ready(function () {
+            render_punch_card("punch_card", [{{ punch_card_data }}]);
+        });
+    </script>
+{% endblock %}
+
+{% block body %}
+<h1>{{ user.user_name }} activity report</h1>
+
+<h2>Contribution summary</h2>
+<ul>
+    <li>Total commits: {{ contribution.commit_count }}</li>
+    <li>Total LOC: {{ contribution.loc }}</li>
+    <li>Review stat (-2, -1, +1, +2): {{ contribution.marks[-2] }}, {{ contribution.marks[-1] }},
+        {{ contribution.marks[1] }}, {{ contribution.marks[2] }}</li>
+    <li>Draft Blueprints: {{ contribution.drafted_blueprint_count }}</li>
+    <li>Completed Blueprints: {{ contribution.completed_blueprint_count }}</li>
+    <li>Emails: {{ contribution.email_count }}</li>
+</ul>
+
+<div id="punch_card" style="width: 100%; height: 350px;"></div>
+
+{% if activity %}
+    {{ show_activity_log(activity, false) }}
+{% endif %}
+
+{% endblock %}

dashboard/web.py

@@ -157,36 +157,7 @@ def get_activity_json(records):
 @decorators.exception_handler()
 @decorators.record_filter(ignore='metric')
 def get_contribution_json(records):
-    marks = dict((m, 0) for m in [-2, -1, 0, 1, 2])
+    return helpers.get_contribution_summary(records)
-    commit_count = 0
-    loc = 0
-    new_blueprint_count = 0
-    competed_blueprint_count = 0
-    email_count = 0
-
-    for record in records:
-        record_type = record['record_type']
-        if record_type == 'commit':
-            commit_count += 1
-            loc += record['loc']
-        elif record['record_type'] == 'mark':
-            marks[record['value']] += 1
-        elif record['record_type'] == 'email':
-            email_count += 1
-        elif record['record_type'] == 'bpd':
-            new_blueprint_count += 1
-        elif record['record_type'] == 'bpc':
-            competed_blueprint_count += 1
-
-    result = {
-        'new_blueprint_count': new_blueprint_count,
-        'competed_blueprint_count': competed_blueprint_count,
-        'commit_count': commit_count,
-        'email_count': email_count,
-        'loc': loc,
-        'marks': marks,
-    }
-    return result
 
 
 @app.route('/api/1.0/companies')
@@ -316,18 +287,7 @@ def get_user(user_id):
     user = vault.get_user_from_runtime_storage(user_id)
     if not user:
         flask.abort(404)
-    user['id'] = user['user_id']
+    user = helpers.extend_user(user)
-    user['text'] = user['user_name']
-    if user['companies']:
-        company_name = user['companies'][-1]['company_name']
-        user['company_link'] = helpers.make_link(
-            company_name, '/', {'company': company_name, 'user_id': ''})
-    else:
-        user['company_link'] = ''
-    if user['emails']:
-        user['gravatar'] = helpers.gravatar(user['emails'][0])
-    else:
-        user['gravatar'] = helpers.gravatar('stackalytics')
     return user