diff --git a/dashboard/memory_storage.py b/dashboard/memory_storage.py index 40badf001..ffe2d7723 100644 --- a/dashboard/memory_storage.py +++ b/dashboard/memory_storage.py @@ -35,14 +35,14 @@ class CachedMemoryStorage(MemoryStorage): self.release_index = {} self.dates = [] for record in records: - self.records[record['record_id']] = record - self.index(record) + self._save_record(record) self.dates = sorted(self.date_index) self.company_name_mapping = dict((c.lower(), c) for c in self.company_index.keys()) - def index(self, record): + def _save_record(self, record): + self.records[record['record_id']] = record self._add_to_index(self.company_index, record, 'company_name') self._add_to_index(self.module_index, record, 'module') self._add_to_index(self.launchpad_id_index, record, 'launchpad_id') @@ -52,6 +52,14 @@ class CachedMemoryStorage(MemoryStorage): record['week'] = user_utils.timestamp_to_week(record['date']) record['loc'] = record['lines_added'] + record['lines_deleted'] + def _remove_record_from_index(self, record): + self.company_index[record['company_name']].remove(record['record_id']) + self.module_index[record['module']].remove(record['record_id']) + self.launchpad_id_index[record['launchpad_id']].remove( + record['record_id']) + self.release_index[record['release']].remove(record['record_id']) + self.date_index[record['date']].remove(record['record_id']) + def _add_to_index(self, record_index, record, key): record_key = record[key] if record_key in record_index: @@ -104,6 +112,12 @@ class CachedMemoryStorage(MemoryStorage): def get_launchpad_ids(self): return self.launchpad_id_index.keys() + def update(self, records): + for record in records: + if record['record_id'] in self.records: + self._remove_record_from_index(record) + self._save_record(record) + def get_memory_storage(memory_storage_type, records): if memory_storage_type == MEMORY_STORAGE_CACHED: diff --git a/dashboard/web.py b/dashboard/web.py index dda275e50..618d77f3c 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -83,6 +83,11 @@ def get_vault(): vault['modules'] = dict((r['module'].lower(), r['project_type'].lower()) for r in modules) app.stackalytics_vault = vault + else: + memory_storage_inst = vault['memory_storage'] + memory_storage_inst.update( + vault['runtime_storage'].get_update(os.getpid())) + return vault diff --git a/stackalytics/processor/runtime_storage.py b/stackalytics/processor/runtime_storage.py index b6d33a1bc..06bb46899 100644 --- a/stackalytics/processor/runtime_storage.py +++ b/stackalytics/processor/runtime_storage.py @@ -115,10 +115,10 @@ class MemcachedStorage(RuntimeStorage): else: for update_id_set in self._make_range(last_update, update_count, BULK_READ_SIZE): - update_set = self.memcached.get_multi(update_id_set, - UPDATE_ID_PREFIX) - for i in self.memcached.get_multi(update_set, - RECORD_ID_PREFIX): + update_set = self.memcached.get_multi( + update_id_set, UPDATE_ID_PREFIX).values() + for i in self.memcached.get_multi( + update_set, RECORD_ID_PREFIX).values(): yield i def active_pids(self, pids): @@ -170,7 +170,7 @@ class MemcachedStorage(RuntimeStorage): self.memcached.set('pids', pids) def _get_record_name(self, record_id): - return RECORD_ID_PREFIX + record_id + return RECORD_ID_PREFIX + str(record_id) def _get_record_count(self): return self.memcached.get('record:count') or 0 @@ -194,7 +194,7 @@ class MemcachedStorage(RuntimeStorage): def _commit_update(self, record_id): count = self._get_update_count() - self.memcached.set(UPDATE_ID_PREFIX + count, record_id) + self.memcached.set(UPDATE_ID_PREFIX + str(count), record_id) self.memcached.set('update:count', count + 1) def _build_index(self):