From d8c382744dfab3d8f8b07b71be95c467dfa45c72 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Wed, 5 Mar 2014 13:34:22 -0500 Subject: [PATCH 1/5] Adding ums query option to nova usage report --- reports/nova_usage_audit.py | 27 +++++++++++++++++++-------- reports/usage_audit.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index b4ed71c..a690e13 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -190,14 +190,18 @@ def _launch_audit_for_period(beginning, ending): return launch_to_exists_fails, new_launches.count(), len(old_launches_dict) -def audit_for_period(beginning, ending): +def audit_for_period(beginning, ending, ums=False): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) + if ums: + verifier_audit_func = usage_audit._verifier_audit_for_day_ums + else: + verifier_audit_func = usage_audit._verifier_audit_for_day + (verify_summary, - verify_detail) = usage_audit._verifier_audit_for_day(beginning_decimal, - ending_decimal, - models.InstanceExists) + verify_detail) = verifier_audit_func(beginning_decimal, ending_decimal, + models.InstanceExists) detail, new_count, old_count = _launch_audit_for_period(beginning_decimal, ending_decimal) @@ -218,14 +222,14 @@ def audit_for_period(beginning, ending): return summary, details -def store_results(start, end, summary, details): +def store_results(start, end, summary, details, ums=False): values = { 'json': make_json_report(summary, details), 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, 'version': 6, - 'name': 'nova usage audit' + 'name': 'nova usage audit' if not ums else 'nova usage audit ums' } report = models.JsonReport(**values) @@ -267,8 +271,15 @@ if __name__ == '__main__': help="Location of the reconciler config file", type=str, default='/etc/stacktach/reconciler-config.json') + parser.add_argument('--ums', + help="Use query to match UMS", + action='store_true') args = parser.parse_args() + if args.ums and args.period_length != 'day': + print "UMS query can only be used with period_length of 'day'." + sys.exit(0) + stacklog.set_default_logger_name('nova_usage_audit') parent_logger = stacklog.get_logger('nova_usage_audit', is_parent=True) log_listener = stacklog.LogListener(parent_logger) @@ -286,9 +297,9 @@ if __name__ == '__main__': start, end = usage_audit.get_previous_period(time, args.period_length) - summary, details = audit_for_period(start, end) + summary, details = audit_for_period(start, end, ums=args.ums) if not args.store: print make_json_report(summary, details) else: - store_results(start, end, summary, details) + store_results(start, end, summary, details, ums=args.ums) diff --git a/reports/usage_audit.py b/reports/usage_audit.py index 4d62ac4..94f9921 100644 --- a/reports/usage_audit.py +++ b/reports/usage_audit.py @@ -1,5 +1,6 @@ import datetime from django.db.models import F +from django.db.models import Q from stacktach import models @@ -101,6 +102,39 @@ def _verifier_audit_for_day(beginning, ending, exists_model): detail.append(['Exist', exist.id, exist.fail_reason]) return summary, detail + +def _verifier_audit_for_day_ums(beginning, ending, exists_model): + summary = {} + + # NOTE(apmelton): + # This is the UMS query we're trying to match. + # where ( + # (created_date between sysdate-1||'12.00.00.000000000 AM' and + # sysdate-1||'04.00.00.000000000 AM' and + # audit_period_begin_timestamp >= sysdate-1||'12.00.00.000000000 AM') + # OR (created_date > sysdate-1||'04.00.00.000000000 AM' and + # audit_period_begin_timestamp < sysdate||'12.00.00.000000000 AM' )) + ums = (Q(raw__when__gte=beginning, raw__when__lte=beginning + (4*60*60), + audit_period_beginning__gte=beginning) | + Q(raw__when__gt=beginning + (4*60*60), + audit_period_beginning__lt=ending)) + + periodic_range = Q(audit_period_ending=(F('audit_period_beginning') + + (60*60*24))) + periodic_exists = exists_model.objects.filter(ums & periodic_range) + summary['periodic'] = _audit_for_exists(periodic_exists) + + instant_range = Q(audit_period_ending__lt=(F('audit_period_beginning') + + (60*60*24))) + instant_exists = exists_model.objects.filter(ums & instant_range) + summary['instantaneous'] = _audit_for_exists(instant_exists) + + failed_query = Q(status=exists_model.FAILED) + failed = exists_model.objects.filter(ums & failed_query) + detail = [['Exist', e.id, e.fail_reason] for e in failed] + return summary, detail + + def get_previous_period(time, period_length): if period_length == 'day': last_period = time - datetime.timedelta(days=1) From a5d4836e1280e62daed73c3aa825b660ee0a4f36 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Wed, 5 Mar 2014 13:52:49 -0500 Subject: [PATCH 2/5] Pulling UMS fencepost offset into script argument --- reports/nova_usage_audit.py | 18 ++++++++++++++---- reports/usage_audit.py | 6 +++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index a690e13..ca11475 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -65,6 +65,8 @@ select stacktach_instancereconcile.id, stacktach_instancereconcile.deleted_at > %s) ) and stacktach_instancereconcile.launched_at < %s;""" +DEFAULT_UMS_OFFSET = 4 * 60 * 60 # 4 Hours + reconciler = None @@ -190,12 +192,14 @@ def _launch_audit_for_period(beginning, ending): return launch_to_exists_fails, new_launches.count(), len(old_launches_dict) -def audit_for_period(beginning, ending, ums=False): +def audit_for_period(beginning, ending, ums=False, ums_offset=0): beginning_decimal = dt.dt_to_decimal(beginning) ending_decimal = dt.dt_to_decimal(ending) if ums: - verifier_audit_func = usage_audit._verifier_audit_for_day_ums + def verifier_audit_func(start, end, model): + return usage_audit._verifier_audit_for_day_ums(start, end, model, + ums_offset) else: verifier_audit_func = usage_audit._verifier_audit_for_day @@ -272,8 +276,13 @@ if __name__ == '__main__': type=str, default='/etc/stacktach/reconciler-config.json') parser.add_argument('--ums', - help="Use query to match UMS", + help="Use query to match UMS, " + "period length of 'day' required.", action='store_true') + parser.add_argument('--ums-offset', + help="UMS' fencepost offset in seconds. Default: 4 days", + type=int, + default=DEFAULT_UMS_OFFSET) args = parser.parse_args() if args.ums and args.period_length != 'day': @@ -297,7 +306,8 @@ if __name__ == '__main__': start, end = usage_audit.get_previous_period(time, args.period_length) - summary, details = audit_for_period(start, end, ums=args.ums) + summary, details = audit_for_period(start, end, ums=args.ums, + ums_offset=args.ums_offset) if not args.store: print make_json_report(summary, details) diff --git a/reports/usage_audit.py b/reports/usage_audit.py index 94f9921..9b606b5 100644 --- a/reports/usage_audit.py +++ b/reports/usage_audit.py @@ -103,7 +103,7 @@ def _verifier_audit_for_day(beginning, ending, exists_model): return summary, detail -def _verifier_audit_for_day_ums(beginning, ending, exists_model): +def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset): summary = {} # NOTE(apmelton): @@ -114,9 +114,9 @@ def _verifier_audit_for_day_ums(beginning, ending, exists_model): # audit_period_begin_timestamp >= sysdate-1||'12.00.00.000000000 AM') # OR (created_date > sysdate-1||'04.00.00.000000000 AM' and # audit_period_begin_timestamp < sysdate||'12.00.00.000000000 AM' )) - ums = (Q(raw__when__gte=beginning, raw__when__lte=beginning + (4*60*60), + ums = (Q(raw__when__gte=beginning, raw__when__lte=beginning + ums_offset, audit_period_beginning__gte=beginning) | - Q(raw__when__gt=beginning + (4*60*60), + Q(raw__when__gt=beginning + ums_offset, audit_period_beginning__lt=ending)) periodic_range = Q(audit_period_ending=(F('audit_period_beginning') + From 8bae0d4cfc8dc8462a2258a48552fc1957b04a0b Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Wed, 5 Mar 2014 14:25:04 -0500 Subject: [PATCH 3/5] Refactor verifier audit functions --- reports/usage_audit.py | 56 ++++++++++++------------------------------ 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/reports/usage_audit.py b/reports/usage_audit.py index 9b606b5..4eed669 100644 --- a/reports/usage_audit.py +++ b/reports/usage_audit.py @@ -70,42 +70,31 @@ def _audit_for_exists(exists_query): return report -def _verifier_audit_for_day(beginning, ending, exists_model): +def _verified_audit_base(base_query, exists_model): summary = {} - filters = { - 'raw__when__gte': beginning, - 'raw__when__lte': ending, - 'audit_period_ending': F('audit_period_beginning') + (60*60*24) - } - periodic_exists = exists_model.objects.filter(**filters) - + periodic_range = Q(audit_period_ending=(F('audit_period_beginning') + + (60*60*24))) + periodic_exists = exists_model.objects.filter(base_query & periodic_range) summary['periodic'] = _audit_for_exists(periodic_exists) - filters = { - 'raw__when__gte': beginning, - 'raw__when__lte': ending, - 'audit_period_ending__lt': F('audit_period_beginning') + (60*60*24) - } - instant_exists = exists_model.objects.filter(**filters) - + instant_range = Q(audit_period_ending__lt=(F('audit_period_beginning') + + (60*60*24))) + instant_exists = exists_model.objects.filter(base_query & instant_range) summary['instantaneous'] = _audit_for_exists(instant_exists) - filters = { - 'raw__when__gte': beginning, - 'raw__when__lte': ending, - 'status': exists_model.FAILED - } - failed = exists_model.objects.filter(**filters) - detail = [] - for exist in failed: - detail.append(['Exist', exist.id, exist.fail_reason]) + failed_query = Q(status=exists_model.FAILED) + failed = exists_model.objects.filter(base_query & failed_query) + detail = [['Exist', e.id, e.fail_reason] for e in failed] return summary, detail -def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset): - summary = {} +def _verifier_audit_for_day(beginning, ending, exists_model): + base_query = Q(raw__when__gte=beginning, raw__when__lte=ending) + return _verified_audit_base(base_query, exists_model) + +def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset): # NOTE(apmelton): # This is the UMS query we're trying to match. # where ( @@ -119,20 +108,7 @@ def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset): Q(raw__when__gt=beginning + ums_offset, audit_period_beginning__lt=ending)) - periodic_range = Q(audit_period_ending=(F('audit_period_beginning') + - (60*60*24))) - periodic_exists = exists_model.objects.filter(ums & periodic_range) - summary['periodic'] = _audit_for_exists(periodic_exists) - - instant_range = Q(audit_period_ending__lt=(F('audit_period_beginning') + - (60*60*24))) - instant_exists = exists_model.objects.filter(ums & instant_range) - summary['instantaneous'] = _audit_for_exists(instant_exists) - - failed_query = Q(status=exists_model.FAILED) - failed = exists_model.objects.filter(ums & failed_query) - detail = [['Exist', e.id, e.fail_reason] for e in failed] - return summary, detail + return _verified_audit_base(ums, exists_model) def get_previous_period(time, period_length): From f0e5b243c657f56488b74edde270e2ac84ade24f Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Wed, 5 Mar 2014 14:34:34 -0500 Subject: [PATCH 4/5] Using partial instead of defining a new func --- reports/nova_usage_audit.py | 7 ++++--- reports/usage_audit.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index ca11475..ae2bd2b 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -20,6 +20,7 @@ import argparse import datetime +import functools import json import sys import os @@ -197,9 +198,9 @@ def audit_for_period(beginning, ending, ums=False, ums_offset=0): ending_decimal = dt.dt_to_decimal(ending) if ums: - def verifier_audit_func(start, end, model): - return usage_audit._verifier_audit_for_day_ums(start, end, model, - ums_offset) + verifier_audit_func = functools.partial( + usage_audit._verifier_audit_for_day_ums, ums_offset=ums_offset + ) else: verifier_audit_func = usage_audit._verifier_audit_for_day diff --git a/reports/usage_audit.py b/reports/usage_audit.py index 4eed669..965d2ac 100644 --- a/reports/usage_audit.py +++ b/reports/usage_audit.py @@ -94,7 +94,7 @@ def _verifier_audit_for_day(beginning, ending, exists_model): return _verified_audit_base(base_query, exists_model) -def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset): +def _verifier_audit_for_day_ums(beginning, ending, exists_model, ums_offset=0): # NOTE(apmelton): # This is the UMS query we're trying to match. # where ( From 670396fce0d1c1a9ccd8c75d02b7d0dcce71fcff Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Thu, 6 Mar 2014 11:48:12 -0500 Subject: [PATCH 5/5] Remove custom name for ums query based report --- reports/nova_usage_audit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reports/nova_usage_audit.py b/reports/nova_usage_audit.py index ae2bd2b..fc4dc9a 100644 --- a/reports/nova_usage_audit.py +++ b/reports/nova_usage_audit.py @@ -227,14 +227,14 @@ def audit_for_period(beginning, ending, ums=False, ums_offset=0): return summary, details -def store_results(start, end, summary, details, ums=False): +def store_results(start, end, summary, details): values = { 'json': make_json_report(summary, details), 'created': dt.dt_to_decimal(datetime.datetime.utcnow()), 'period_start': start, 'period_end': end, 'version': 6, - 'name': 'nova usage audit' if not ums else 'nova usage audit ums' + 'name': 'nova usage audit' } report = models.JsonReport(**values) @@ -313,4 +313,4 @@ if __name__ == '__main__': if not args.store: print make_json_report(summary, details) else: - store_results(start, end, summary, details, ums=args.ums) + store_results(start, end, summary, details)