Merge pull request #303 from rackerlabs/master

Promote Master to Stable

reports/nova_usage_audit.py

@@ -20,6 +20,7 @@
 
 import argparse
 import datetime
+import functools
 import json
 import sys
 import os
@@ -65,6 +66,8 @@ select stacktach_instancereconcile.id,
                 stacktach_instancereconcile.deleted_at > %s)
         ) and stacktach_instancereconcile.launched_at < %s;"""
 
+DEFAULT_UMS_OFFSET = 4 * 60 * 60  # 4 Hours
+
 reconciler = None
 
 
@@ -190,14 +193,20 @@ def _launch_audit_for_period(beginning, ending):
     return launch_to_exists_fails, new_launches.count(), len(old_launches_dict)
 
 
-def audit_for_period(beginning, ending):
+def audit_for_period(beginning, ending, ums=False, ums_offset=0):
     beginning_decimal = dt.dt_to_decimal(beginning)
     ending_decimal = dt.dt_to_decimal(ending)
 
+    if ums:
+        verifier_audit_func = functools.partial(
+            usage_audit._verifier_audit_for_day_ums, ums_offset=ums_offset
+        )
+    else:
+        verifier_audit_func = usage_audit._verifier_audit_for_day
+
     (verify_summary,
-     verify_detail) = usage_audit._verifier_audit_for_day(beginning_decimal,
+     verify_detail) = verifier_audit_func(beginning_decimal, ending_decimal,
-                                                          ending_decimal,
+                                          models.InstanceExists)
-                                                          models.InstanceExists)
     detail, new_count, old_count = _launch_audit_for_period(beginning_decimal,
                                                             ending_decimal)
 
@@ -267,8 +276,20 @@ if __name__ == '__main__':
                         help="Location of the reconciler config file",
                         type=str,
                         default='/etc/stacktach/reconciler-config.json')
+    parser.add_argument('--ums',
+                        help="Use query to match UMS, "
+                             "period length of 'day' required.",
+                        action='store_true')
+    parser.add_argument('--ums-offset',
+                        help="UMS' fencepost offset in seconds. Default: 4 days",
+                        type=int,
+                        default=DEFAULT_UMS_OFFSET)
     args = parser.parse_args()
 
+    if args.ums and args.period_length != 'day':
+        print "UMS query can only be used with period_length of 'day'."
+        sys.exit(0)
+
     stacklog.set_default_logger_name('nova_usage_audit')
     parent_logger = stacklog.get_logger('nova_usage_audit', is_parent=True)
     log_listener = stacklog.LogListener(parent_logger)
@@ -286,7 +307,8 @@ if __name__ == '__main__':
 
     start, end = usage_audit.get_previous_period(time, args.period_length)
 
-    summary, details = audit_for_period(start, end)
+    summary, details = audit_for_period(start, end, ums=args.ums,
+                                        ums_offset=args.ums_offset)
 
     if not args.store:
         print make_json_report(summary, details)

reports/usage_audit.py

@@ -1,5 +1,6 @@
 import datetime
 from django.db.models import F
+from django.db.models import Q
 from stacktach import models
 
 
@@ -69,38 +70,47 @@ def _audit_for_exists(exists_query):
     return report
 
 
-def _verifier_audit_for_day(beginning, ending, exists_model):
+def _verified_audit_base(base_query, exists_model):
     summary = {}
 
-    filters = {
+    periodic_range = Q(audit_period_ending=(F('audit_period_beginning') +
-        'raw__when__gte': beginning,
+                                            (60*60*24)))
-        'raw__when__lte': ending,
+    periodic_exists = exists_model.objects.filter(base_query & periodic_range)
-        'audit_period_ending': F('audit_period_beginning') + (60*60*24)
-    }
-    periodic_exists = exists_model.objects.filter(**filters)
-
     summary['periodic'] = _audit_for_exists(periodic_exists)
 
-    filters = {
+    instant_range = Q(audit_period_ending__lt=(F('audit_period_beginning') +
-        'raw__when__gte': beginning,
+                                               (60*60*24)))
-        'raw__when__lte': ending,
+    instant_exists = exists_model.objects.filter(base_query & instant_range)
-        'audit_period_ending__lt': F('audit_period_beginning') + (60*60*24)
-    }
-    instant_exists = exists_model.objects.filter(**filters)
-
     summary['instantaneous'] = _audit_for_exists(instant_exists)
 
-    filters = {
+    failed_query = Q(status=exists_model.FAILED)
-        'raw__when__gte': beginning,
+    failed = exists_model.objects.filter(base_query & failed_query)
-        'raw__when__lte': ending,
+    detail = [['Exist', e.id, e.fail_reason] for e in failed]
-        'status': exists_model.FAILED
-    }
-    failed = exists_model.objects.filter(**filters)
-    detail = []
-    for exist in failed:
-        detail.append(['Exist', exist.id, exist.fail_reason])
     return summary, detail
 
+
+def _verifier_audit_for_day(beginning, ending, exists_model):
+    base_query = Q(raw__when__gte=beginning, raw__when__lte=ending)
+    return _verified_audit_base(base_query, exists_model)
+
+
+def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset=0):
+    # NOTE(apmelton):
+    # This is the UMS query we're trying to match.
+    # where (
+    #     (created_date between sysdate-1||'12.00.00.000000000 AM' and
+    #                           sysdate-1||'04.00.00.000000000 AM' and
+    #      audit_period_begin_timestamp >= sysdate-1||'12.00.00.000000000 AM')
+    # OR (created_date > sysdate-1||'04.00.00.000000000 AM' and
+    #     audit_period_begin_timestamp < sysdate||'12.00.00.000000000 AM' ))
+    ums = (Q(raw__when__gte=beginning, raw__when__lte=beginning + ums_offset,
+             audit_period_beginning__gte=beginning) |
+           Q(raw__when__gt=beginning + ums_offset,
+             audit_period_beginning__lt=ending))
+
+    return _verified_audit_base(ums, exists_model)
+
+
 def get_previous_period(time, period_length):
     if period_length == 'day':
         last_period = time - datetime.timedelta(days=1)