Commits correction framework is implemented

* Introduced new parameter in config file
* Added a notice for changed commits to UI
Also:
* Fixed default config param for sources_root
* Removed background image from layout

Implements blueprint commits-corrections-framework

Change-Id: I3c2c76a45ed75aaf67671b02649ba6abc24be083

dashboard/static/css/style.css

@@ -1,7 +1,6 @@
 html, body {
     font-family: 'PT Sans', arial, sans-serif;
     font-size: 14px;
-    background: url(../images/osstats_tile.jpg) repeat-x;
     height: 100%;
     color: #41454d;
     margin: 0;

dashboard/templates/engineer_details.html

@@ -33,6 +33,9 @@
         <div>
             <h4>{{ rec.date|datetimeformat }} to <a href="https://launchpad.net/{{ rec.module }}">{{ rec.module }}</a>
             </h4>
+            {% if rec.correction_comment %}
+                <div style='font-weight: bold; color: red; padding-left: 2em;'>Commit corrected: {{ rec.correction_comment }}</div>
+            {% endif %}
             <div style='font-weight: bold; padding-left: 2em;'>{{ rec.subject }}</div>
             <div style='white-space: pre-wrap; padding-left: 2em;'>{{ rec|commit_message|safe }}</div>
             <div style="padding-left: 2em;"><span style="color: green">+ {{ rec.lines_added }}</span>

dashboard/templates/module_details.html

@@ -56,6 +56,9 @@
                 <em>{{ rec.date|datetimeformat }}</em>
                 </div>
 
+                {% if rec.correction_comment %}
+                    <div style='font-weight: bold; color: red; padding-left: 2em;'>Commit corrected: {{ rec.correction_comment }}</div>
+                {% endif %}
                 <div><b>{{ rec.subject }}</b></div>
                 <div style="white-space: pre-wrap;">{{ rec|commit_message|safe }}</div>
                 <div><span style="color: green">+ {{ rec.lines_added }}</span>

etc/corrections.json

@@ -2,7 +2,7 @@
     "corrections": [
         {
             "commit_id": "ee3fe4e836ca1c81e50a8324a9b5f982de4fa97f",
-            "correction_comment": "Rename of Quantum to Neutron",
+            "correction_comment": "Reset LOC to 0",
             "lines_added": 0,
             "lines_deleted": 0
         }

etc/stackalytics.conf

@@ -6,7 +6,7 @@
 # default-data = /etc/stackalytics/default_data.json
 
 # The folder that holds all project sources to analyze
-# sources_root = /var/run/stackalytics
+# sources_root = /var/local/stackalytics
 
 # Runtime storage URI
 # runtime_storage_uri = memcached://127.0.0.1:11211
@@ -22,3 +22,6 @@
 
 # Port where dashboard listens on
 # listen_port = 8080
+
+# The address of file with corrections data
+# corrections-uri = https://raw.github.com/stackforge/stackalytics/master/etc/corrections.json

stackalytics/processor/config.py

@@ -36,4 +36,8 @@ OPTS = [
                help='The address dashboard listens on'),
     cfg.IntOpt('listen-port', default=8080,
                help='The port dashboard listens on'),
+    cfg.StrOpt('corrections-uri',
+               default=('https://raw.github.com/stackforge/stackalytics/'
+                        'master/etc/corrections.json'),
+               help='The address of file with corrections data'),
 ]

stackalytics/processor/main.py

@@ -12,10 +12,12 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import json
 
 from oslo.config import cfg
 import psutil
 from psutil import _error
+import urllib2
 
 from stackalytics.openstack.common import log as logging
 from stackalytics.processor import commit_processor
@@ -84,6 +86,13 @@ def update_repos(runtime_storage, persistent_storage):
         process_repo(repo, runtime_storage, processor)
 
 
+def apply_corrections(uri, runtime_storage_inst):
+    corrections_fd = urllib2.urlopen(uri)
+    raw = corrections_fd.read()
+    corrections_fd.close()
+    runtime_storage_inst.apply_corrections(json.loads(raw)['corrections'])
+
+
 def main():
     # init conf and logging
     conf = cfg.CONF
@@ -111,6 +120,8 @@ def main():
 
     update_repos(runtime_storage_inst, persistent_storage_inst)
 
+    apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
+
 
 if __name__ == '__main__':
     main()

stackalytics/processor/runtime_storage.py

@@ -34,6 +34,9 @@ class RuntimeStorage(object):
     def set_records(self, records_iterator):
         pass
 
+    def apply_corrections(self, corrections_iterator):
+        pass
+
     def get_head_commit_id(self, uri, branch):
         pass
 
@@ -65,12 +68,10 @@ class MemcachedStorage(RuntimeStorage):
             if record['commit_id'] in self.commit_id_index:
                 # update
                 record_id = self.commit_id_index[record['commit_id']]
-                old_record = self.memcached.get(
+                original = self.memcached.get(self._get_record_name(record_id))
-                    self._get_record_name(record_id))
+                original['branches'] |= record['branches']
-                old_record['branches'] |= record['branches']
                 LOG.debug('Update record %s' % record)
-                self.memcached.set(self._get_record_name(record_id),
+                self.memcached.set(self._get_record_name(record_id), original)
-                                   old_record)
             else:
                 # insert record
                 record_id = self._get_record_count()
@@ -81,6 +82,24 @@ class MemcachedStorage(RuntimeStorage):
 
             self._commit_update(record_id)
 
+    def apply_corrections(self, corrections_iterator):
+        for correction in corrections_iterator:
+            if correction['commit_id'] not in self.commit_id_index:
+                continue
+
+            record_id = self.commit_id_index[correction['commit_id']]
+            original = self.memcached.get(self._get_record_name(record_id))
+            need_update = False
+
+            for field, value in correction.iteritems():
+                if (field not in original) or (original[field] != value):
+                    need_update = True
+                    original[field] = value
+
+            if need_update:
+                self.memcached.set(self._get_record_name(record_id), original)
+                self._commit_update(record_id)
+
     def get_head_commit_id(self, uri, branch):
         key = str(urllib.quote_plus(uri) + ':' + branch)
         return self.memcached.get(key)