From 45ae991f66159a4b5281495f8c4b145284be8e77 Mon Sep 17 00:00:00 2001
From: Andrew Melton <andrew.melton@rackspace.com>
Date: Thu, 21 Mar 2013 16:18:37 -0400
Subject: [PATCH] Updating migrations to work with optimized usage parsing
---
... => 006_populate_rebuilds_from_rawdata.py} | 11 ++++-
....py => 006_populate_usage_from_rawdata.py} | 15 +++++--
tests/unit/test_stacktach.py | 43 -------------------
3 files changed, 21 insertions(+), 48 deletions(-)
rename migrations/{006_populate_rebuilds.py => 006_populate_rebuilds_from_rawdata.py} (83%)
rename migrations/{006_populate_all_usage.py => 006_populate_usage_from_rawdata.py} (91%)
diff --git a/migrations/006_populate_rebuilds.py b/migrations/006_populate_rebuilds_from_rawdata.py
similarity index 83%
rename from migrations/006_populate_rebuilds.py
rename to migrations/006_populate_rebuilds_from_rawdata.py
index b912c04..a2e9161 100644
--- a/migrations/006_populate_rebuilds.py
+++ b/migrations/006_populate_rebuilds_from_rawdata.py
@@ -4,6 +4,14 @@ import datetime
import os
import sys
+try:
+ import ujson as json
+except ImportError:
+ try:
+ import simplejson as json
+ except ImportError:
+ import json
+
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir, os.pardir))
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
@@ -26,7 +34,8 @@ def add_past_usage(raws):
print "%s events to be processed" % count
last_update = datetime.datetime.utcnow()
for raw in raws:
- views.aggregate_usage(raw)
+ json_dict = json.dumps(raw.json)
+ views.aggregate_usage(raw, json_dict[1])
processed += 1
if processed % 50 == 0:
next_update = last_update + datetime.timedelta(seconds=30)
diff --git a/migrations/006_populate_all_usage.py b/migrations/006_populate_usage_from_rawdata.py
similarity index 91%
rename from migrations/006_populate_all_usage.py
rename to migrations/006_populate_usage_from_rawdata.py
index b7edd60..813696e 100644
--- a/migrations/006_populate_all_usage.py
+++ b/migrations/006_populate_usage_from_rawdata.py
@@ -4,7 +4,13 @@ import datetime
import os
import sys
-import multiprocessing
+try:
+ import ujson as json
+except ImportError:
+ try:
+ import simplejson as json
+ except ImportError:
+ import json
POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir, os.pardir))
@@ -48,9 +54,9 @@ def usage_already_exists(raw):
return False
-def populate_usage(raw):
+def populate_usage(raw, body):
if not usage_already_exists(raw):
- views.aggregate_usage(raw)
+ views.aggregate_usage(raw, body)
def print_status(event, completed, errored, total):
@@ -86,7 +92,8 @@ for event in events:
new_loc = loc + 500
for raw in raws[loc:new_loc]:
try:
- populate_usage(raw)
+ json_dict = json.dumps(raw.json)
+ populate_usage(raw, json_dict[1])
completed += 1
except Exception:
errored += 1
diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py
index 212cee1..102f82b 100644
--- a/tests/unit/test_stacktach.py
+++ b/tests/unit/test_stacktach.py
@@ -421,22 +421,6 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
self.assertEquals(usage.instance_type_id, '1')
self.mox.VerifyAll()
- def test_process_usage_for_new_launch(self):
- when = utils.decimal_utc()
- notif = utils.create_nova_notif(request_id=REQUEST_ID_1)
- json_str = json.dumps(notif)
- event = 'compute.instance.rebuild.start'
- raw = utils.create_raw(self.mox, when, event=event, json_str=json_str)
- usage = self.mox.CreateMockAnything()
- views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
- request_id=REQUEST_ID_1) \
- .AndReturn((usage, True))
- views.STACKDB.save(usage)
- self.mox.ReplayAll()
- views._process_usage_for_new_launch(raw)
- self.assertEquals(usage.instance_type_id, '1')
- self.mox.VerifyAll()
-
def test_process_usage_for_updates_create_end(self):
when_time = datetime.datetime.utcnow()
when_str = str(when_time)
@@ -464,33 +448,6 @@ class StacktackUsageParsingTestCase(unittest.TestCase):
self.assertEqual(usage.launched_at, when_decimal)
self.mox.VerifyAll()
- def test_process_usage_for_updates_create_end(self):
- when_time = datetime.datetime.utcnow()
- when_str = str(when_time)
- when_decimal = utils.decimal_utc(when_time)
- notif = utils.create_nova_notif(request_id=REQUEST_ID_1,
- launched=str(when_time))
- json_str = json.dumps(notif)
- event = 'compute.instance.rebuild.end'
- raw = utils.create_raw(self.mox, when_decimal, event=event,
- json_str=json_str)
- usage = self.mox.CreateMockAnything()
- usage.instance = INSTANCE_ID_1
- usage.request_id = REQUEST_ID_1
- usage.instance_type_id = '1'
- views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1,
- request_id=REQUEST_ID_1) \
- .AndReturn((usage, True))
- views.STACKDB.save(usage)
- self.mox.ReplayAll()
-
- views._process_usage_for_updates(raw)
- self.assertEqual(usage.instance, INSTANCE_ID_1)
- self.assertEqual(usage.request_id, REQUEST_ID_1)
- self.assertEqual(usage.instance_type_id, '1')
- self.assertEqual(usage.launched_at, when_decimal)
- self.mox.VerifyAll()
-
def test_process_usage_for_updates_revert_end(self):
when_time = datetime.datetime.utcnow()
when_decimal = utils.decimal_utc(when_time)