x/stackalytics
Code Issues Proposed changes

Commits correction framework is implemented

Browse Source 
* Introduced new parameter in config file
* Added a notice for changed commits to UI
Also:
* Fixed default config param for sources_root
* Removed background image from layout

Implements blueprint commits-corrections-framework

Change-Id: I3c2c76a45ed75aaf67671b02649ba6abc24be083
This commit is contained in:
Ilya Shakhat 2013-07-17 14:13:38 +04:00
parent 91c0254b5b
commit b3bfb7b7f2
8 changed files with 50 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dashboard/static/css/style.css
View File

@ -1,7 +1,6 @@
html, body {
font-family: 'PT Sans', arial, sans-serif;
font-size: 14px;
background: url(../images/osstats_tile.jpg) repeat-x;
height: 100%;
color: #41454d;
margin: 0;

3
dashboard/templates/engineer_details.html
View File

@ -33,6 +33,9 @@
<div>
<h4>{{ rec.date|datetimeformat }} to <a href="https://launchpad.net/{{ rec.module }}">{{ rec.module }}</a>
</h4>
{% if rec.correction_comment %}
<div style='font-weight: bold; color: red; padding-left: 2em;'>Commit corrected: {{ rec.correction_comment }}</div>
{% endif %}
<div style='font-weight: bold; padding-left: 2em;'>{{ rec.subject }}</div>
<div style='white-space: pre-wrap; padding-left: 2em;'>{{ rec|commit_message|safe }}</div>
<div style="padding-left: 2em;"><span style="color: green">+ {{ rec.lines_added }}</span>

3
dashboard/templates/module_details.html
View File

@ -56,6 +56,9 @@
<em>{{ rec.date|datetimeformat }}</em>
</div>
{% if rec.correction_comment %}
<div style='font-weight: bold; color: red; padding-left: 2em;'>Commit corrected: {{ rec.correction_comment }}</div>
{% endif %}
<div><b>{{ rec.subject }}</b></div>
<div style="white-space: pre-wrap;">{{ rec|commit_message|safe }}</div>
<div><span style="color: green">+ {{ rec.lines_added }}</span>

2
etc/corrections.json
View File

@ -2,7 +2,7 @@
"corrections": [
{
"commit_id": "ee3fe4e836ca1c81e50a8324a9b5f982de4fa97f",
"correction_comment": "Rename of Quantum to Neutron",
"correction_comment": "Reset LOC to 0",
"lines_added": 0,
"lines_deleted": 0
}

5
etc/stackalytics.conf
View File

@ -6,7 +6,7 @@
# default-data = /etc/stackalytics/default_data.json
# The folder that holds all project sources to analyze
# sources_root = /var/run/stackalytics
# sources_root = /var/local/stackalytics
# Runtime storage URI
# runtime_storage_uri = memcached://127.0.0.1:11211
@ -22,3 +22,6 @@
# Port where dashboard listens on
# listen_port = 8080
# The address of file with corrections data
# corrections-uri = https://raw.github.com/stackforge/stackalytics/master/etc/corrections.json

4
stackalytics/processor/config.py
View File

@ -36,4 +36,8 @@ OPTS = [
help='The address dashboard listens on'),
cfg.IntOpt('listen-port', default=8080,
help='The port dashboard listens on'),
cfg.StrOpt('corrections-uri',
default=('https://raw.github.com/stackforge/stackalytics/'
'master/etc/corrections.json'),
help='The address of file with corrections data'),
]

11
stackalytics/processor/main.py
View File

@ -12,10 +12,12 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from oslo.config import cfg
import psutil
from psutil import _error
import urllib2
from stackalytics.openstack.common import log as logging
from stackalytics.processor import commit_processor
@ -84,6 +86,13 @@ def update_repos(runtime_storage, persistent_storage):
process_repo(repo, runtime_storage, processor)
def apply_corrections(uri, runtime_storage_inst):
corrections_fd = urllib2.urlopen(uri)
raw = corrections_fd.read()
corrections_fd.close()
runtime_storage_inst.apply_corrections(json.loads(raw)['corrections'])
def main():
# init conf and logging
conf = cfg.CONF
@ -111,6 +120,8 @@ def main():
update_repos(runtime_storage_inst, persistent_storage_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
if __name__ == '__main__':
main()

29
stackalytics/processor/runtime_storage.py
View File

@ -34,6 +34,9 @@ class RuntimeStorage(object):
def set_records(self, records_iterator):
pass
def apply_corrections(self, corrections_iterator):
pass
def get_head_commit_id(self, uri, branch):
pass
@ -65,12 +68,10 @@ class MemcachedStorage(RuntimeStorage):
if record['commit_id'] in self.commit_id_index:
# update
record_id = self.commit_id_index[record['commit_id']]
old_record = self.memcached.get(
self._get_record_name(record_id))
old_record['branches'] |= record['branches']
original = self.memcached.get(self._get_record_name(record_id))
original['branches'] |= record['branches']
LOG.debug('Update record %s' % record)
self.memcached.set(self._get_record_name(record_id),
old_record)
self.memcached.set(self._get_record_name(record_id), original)
else:
# insert record
record_id = self._get_record_count()
@ -81,6 +82,24 @@ class MemcachedStorage(RuntimeStorage):
self._commit_update(record_id)
def apply_corrections(self, corrections_iterator):
for correction in corrections_iterator:
if correction['commit_id'] not in self.commit_id_index:
continue
record_id = self.commit_id_index[correction['commit_id']]
original = self.memcached.get(self._get_record_name(record_id))
need_update = False
for field, value in correction.iteritems():
if (field not in original) or (original[field] != value):
need_update = True
original[field] = value
if need_update:
self.memcached.set(self._get_record_name(record_id), original)
self._commit_update(record_id)
def get_head_commit_id(self, uri, branch):
key = str(urllib.quote_plus(uri) + ':' + branch)
return self.memcached.get(key)