diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index b4ed71c..fc4dc9a 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -20,6 +20,7 @@ import argparse import datetime +import functools import json import sys import os @@ -65,6 +66,8 @@ select stacktach_instancereconcile.id, stacktach_instancereconcile.deleted_at > %s) ) and stacktach_instancereconcile.launched_at < %s;""" +DEFAULT_UMS_OFFSET = 4 * 60 * 60 # 4 Hours + reconciler = None @@ -190,14 +193,20 @@ def _launch_audit_for_period(beginning, ending): return launch_to_exists_fails, new_launches.count(), len(old_launches_dict) -def audit_for_period(beginning, ending): +def audit_for_period(beginning, ending, ums=False, ums_offset=0): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) + if ums: + verifier_audit_func = functools.partial( + usage_audit._verifier_audit_for_day_ums, ums_offset=ums_offset + ) + else: + verifier_audit_func = usage_audit._verifier_audit_for_day + (verify_summary, - verify_detail) = usage_audit._verifier_audit_for_day(beginning_decimal, - ending_decimal, - models.InstanceExists) + verify_detail) = verifier_audit_func(beginning_decimal, ending_decimal, + models.InstanceExists) detail, new_count, old_count = _launch_audit_for_period(beginning_decimal, ending_decimal) @@ -267,8 +276,20 @@ if __name__ == '__main__': help="Location of the reconciler config file", type=str, default='/etc/stacktach/reconciler-config.json') + parser.add_argument('--ums', + help="Use query to match UMS, " + "period length of 'day' required.", + action='store_true') + parser.add_argument('--ums-offset', + help="UMS' fencepost offset in seconds. Default: 4 days", + type=int, + default=DEFAULT_UMS_OFFSET) args = parser.parse_args() + if args.ums and args.period_length != 'day': + print "UMS query can only be used with period_length of 'day'." + sys.exit(0) + stacklog.set_default_logger_name('nova_usage_audit') parent_logger = stacklog.get_logger('nova_usage_audit', is_parent=True) log_listener = stacklog.LogListener(parent_logger) @@ -286,7 +307,8 @@ if __name__ == '__main__': start, end = usage_audit.get_previous_period(time, args.period_length) - summary, details = audit_for_period(start, end) + summary, details = audit_for_period(start, end, ums=args.ums, + ums_offset=args.ums_offset) if not args.store: print make_json_report(summary, details) diff --git a/reports/usage_audit.py b/reports/usage_audit.py index 4d62ac4..965d2ac 100644 --- a/reports/usage_audit.py +++ b/reports/usage_audit.py @@ -1,5 +1,6 @@ import datetime from django.db.models import F +from django.db.models import Q from stacktach import models @@ -69,38 +70,47 @@ def _audit_for_exists(exists_query): return report -def _verifier_audit_for_day(beginning, ending, exists_model): +def _verified_audit_base(base_query, exists_model): summary = {} - filters = { - 'raw__when__gte': beginning, - 'raw__when__lte': ending, - 'audit_period_ending': F('audit_period_beginning') + (60*60*24) - } - periodic_exists = exists_model.objects.filter(**filters) - + periodic_range = Q(audit_period_ending=(F('audit_period_beginning') + + (60*60*24))) + periodic_exists = exists_model.objects.filter(base_query & periodic_range) summary['periodic'] = _audit_for_exists(periodic_exists) - filters = { - 'raw__when__gte': beginning, - 'raw__when__lte': ending, - 'audit_period_ending__lt': F('audit_period_beginning') + (60*60*24) - } - instant_exists = exists_model.objects.filter(**filters) - + instant_range = Q(audit_period_ending__lt=(F('audit_period_beginning') + + (60*60*24))) + instant_exists = exists_model.objects.filter(base_query & instant_range) summary['instantaneous'] = _audit_for_exists(instant_exists) - filters = { - 'raw__when__gte': beginning, - 'raw__when__lte': ending, - 'status': exists_model.FAILED - } - failed = exists_model.objects.filter(**filters) - detail = [] - for exist in failed: - detail.append(['Exist', exist.id, exist.fail_reason]) + failed_query = Q(status=exists_model.FAILED) + failed = exists_model.objects.filter(base_query & failed_query) + detail = [['Exist', e.id, e.fail_reason] for e in failed] return summary, detail + +def _verifier_audit_for_day(beginning, ending, exists_model): + base_query = Q(raw__when__gte=beginning, raw__when__lte=ending) + return _verified_audit_base(base_query, exists_model) + + +def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset=0): + # NOTE(apmelton): + # This is the UMS query we're trying to match. + # where ( + # (created_date between sysdate-1||'12.00.00.000000000 AM' and + # sysdate-1||'04.00.00.000000000 AM' and + # audit_period_begin_timestamp >= sysdate-1||'12.00.00.000000000 AM') + # OR (created_date > sysdate-1||'04.00.00.000000000 AM' and + # audit_period_begin_timestamp < sysdate||'12.00.00.000000000 AM' )) + ums = (Q(raw__when__gte=beginning, raw__when__lte=beginning + ums_offset, + audit_period_beginning__gte=beginning) | + Q(raw__when__gt=beginning + ums_offset, + audit_period_beginning__lt=ending)) + + return _verified_audit_base(ums, exists_model) + + def get_previous_period(time, period_length): if period_length == 'day': last_period = time - datetime.timedelta(days=1)