x/stacktach
Code Issues Proposed changes

Merge pull request #210 from manalilatkar/glance_usage_audit

Browse Source 
Glance Usage Audit
This commit is contained in:
Andrew Melton 2013-11-04 06:41:07 -08:00
commit 660ebbd7a5
4 changed files with 333 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
reports/__init__.py Normal file
View File

221
reports/glance_usage_audit.py Normal file
View File

@ -0,0 +1,221 @@
import argparse
import datetime
import json
import os
import sys
sys.path.append(os.environ.get('STACKTACH_INSTALL_DIR', '/stacktach'))
from django.db.models import F
from reports import usage_audit
from stacktach import models
from stacktach import datetime_to_decimal as dt
OLD_IMAGES_QUERY = """
select * from stacktach_imageusage left join stacktach_imagedeletes
on (stacktach_imageusage.uuid = stacktach_imagedeletes.uuid and
deleted_at < %s)
where stacktach_imagedeletes.id IS NULL
and created_at is not null and created_at < %s;"""
def audit_usages_to_exists(exists, usages):
# checks if all exists correspond to the given usages
fails = []
for (uuid, images) in usages.items():
if uuid not in exists:
msg = "No exists for usage (%s)" % uuid
fails.append(['Usage', images[0]['id'], msg])
return fails
def _get_new_images(beginning, ending):
filters = {
'created_at__gte': beginning,
'created_at__lte': ending,
}
return models.ImageUsage.objects.filter(**filters)
def _get_exists(beginning, ending):
filters = {
'audit_period_beginning': beginning,
'audit_period_ending__gte': beginning,
'audit_period_ending__lte': ending,
}
return models.ImageExists.objects.filter(**filters)
def valid_datetime(d):
try:
t = datetime.datetime.strptime(d, "%Y-%m-%d %H:%M:%S")
return t
except Exception, e:
raise argparse.ArgumentTypeError(
"'%s' is not in YYYY-MM-DD HH:MM:SS format." % d)
def audit_for_period(beginning, ending):
beginning_decimal = dt.dt_to_decimal(beginning)
ending_decimal = dt.dt_to_decimal(ending)
(verify_summary,
verify_detail) = _verifier_audit_for_day(beginning_decimal,
ending_decimal,
models.ImageExists)
detail, new_count, old_count = _image_audit_for_period(beginning_decimal,
ending_decimal)
summary = {
'verifier': verify_summary,
'image_summary': {
'new_images': new_count,
'old_images': old_count,
'failures': len(detail)
},
}
details = {
'exist_fails': verify_detail,
'image_fails': detail,
}
return summary, details
def _verifier_audit_for_day(beginning, ending, exists_model):
summary = {}
period = 60*60*24-0.000001
if args.period_length == 'hour':
period = 60*60-0.000001
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'audit_period_ending': F('audit_period_beginning') + period
}
exists = exists_model.objects.filter(**filters)
summary['exists'] = _audit_for_exists(exists)
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'status': exists_model.FAILED
}
failed = exists_model.objects.filter(**filters)
detail = []
for exist in failed:
detail.append(['Exist', exist.id, exist.fail_reason])
return summary, detail
def _audit_for_exists(exists_query):
(verified, reconciled,
fail, pending, verifying) = usage_audit._status_queries(exists_query)
(success, unsent, redirect,
client_error, server_error) = usage_audit._send_status_queries(verified)
report = {
'count': exists_query.count(),
'verified': verified.count(),
'failed': fail.count(),
'pending': pending.count(),
'verifying': verifying.count(),
'send_status': {
'success': success.count(),
'unsent': unsent.count(),
'redirect': redirect.count(),
'client_error': client_error.count(),
'server_error': server_error.count(),
}
}
return report
def _image_audit_for_period(beginning, ending):
images_dict = {}
new_images = _get_new_images(beginning, ending)
for image in new_images:
uuid = image.uuid
l = {'id': image.id, 'created_at': image.created_at}
if uuid in images_dict:
images_dict[uuid].append(l)
else:
images_dict[uuid] = [l, ]
# Django's safe substitution doesn't allow dict substitution...
# Thus, we send it 'beginning' two times...
old_images = models.ImageUsage.objects\
.raw(OLD_IMAGES_QUERY,
[beginning, beginning])
old_images_dict = {}
for image in old_images:
uuid = image.uuid
l = {'id': image.id, 'created_at': image.created_at}
old_images_dict[uuid] = l
exists_dict = {}
exists = _get_exists(beginning, ending)
for exist in exists:
uuid = exist.uuid
e = {'id': exist.id,
'created_at': exist.created_at,
'deleted_at': exist.deleted_at}
if uuid in exists_dict:
exists_dict[uuid].append(e)
else:
exists_dict[uuid] = [e, ]
image_to_exists_fails = audit_usages_to_exists(exists_dict,images_dict)
return image_to_exists_fails, new_images.count(), len(old_images_dict)
def store_results(start, end, summary, details):
values = {
'json': make_json_report(summary, details),
'created': dt.dt_to_decimal(datetime.datetime.utcnow()),
'period_start': start,
'period_end': end,
'version': 4,
'name': 'glance usage audit'
}
report = models.JsonReport(**values)
report.save()
def make_json_report(summary, details):
report = [{'summary': summary},
['Object', 'ID', 'Error Description']]
report.extend(details['exist_fails'])
report.extend(details['image_fails'])
return json.dumps(report)
if __name__ == '__main__':
parser = argparse.ArgumentParser('StackTach Nova Usage Audit Report')
parser.add_argument('--period_length',
choices=['hour', 'day'], default='day')
parser.add_argument('--utcdatetime',
help="Override the end time used to generate report.",
type=valid_datetime, default=None)
parser.add_argument('--store',
help="If set to true, report will be stored. "
"Otherwise, it will just be printed",
type=bool, default=False)
args = parser.parse_args()
if args.utcdatetime is not None:
time = args.utcdatetime
else:
time = datetime.datetime.utcnow()
start, end = usage_audit.get_previous_period(time, args.period_length)
summary, details = audit_for_period(start, end)
if not args.store:
print make_json_report(summary, details)
else:
store_results(start, end, summary, details)

125
reports/nova_usage_audit.py
View File

@ -26,7 +26,7 @@ import os
sys.path.append(os.environ.get('STACKTACH_INSTALL_DIR', '/stacktach'))
from django.db.models import F
import usage_audit
from stacktach import datetime_to_decimal as dt
from stacktach import models
@ -123,98 +123,6 @@ def _audit_launches_to_exists(launches, exists, beginning):
return fails
def _status_queries(exists_query):
verified = exists_query.filter(status=models.InstanceExists.VERIFIED)
reconciled = exists_query.filter(status=models.InstanceExists.RECONCILED)
fail = exists_query.filter(status=models.InstanceExists.FAILED)
pending = exists_query.filter(status=models.InstanceExists.PENDING)
verifying = exists_query.filter(status=models.InstanceExists.VERIFYING)
return verified, reconciled, fail, pending, verifying
def _send_status_queries(exists_query):
unsent = exists_query.filter(send_status=0)
success = exists_query.filter(send_status__gte=200,
send_status__lt=300)
redirect = exists_query.filter(send_status__gte=300,
send_status__lt=400)
client_error = exists_query.filter(send_status__gte=400,
send_status__lt=500)
server_error = exists_query.filter(send_status__gte=500,
send_status__lt=600)
return success, unsent, redirect, client_error, server_error
def _audit_for_exists(exists_query):
(verified, reconciled,
fail, pending, verifying) = _status_queries(exists_query)
(success, unsent, redirect,
client_error, server_error) = _send_status_queries(verified)
(success_rec, unsent_rec, redirect_rec,
client_error_rec, server_error_rec) = _send_status_queries(reconciled)
report = {
'count': exists_query.count(),
'verified': verified.count(),
'reconciled': reconciled.count(),
'failed': fail.count(),
'pending': pending.count(),
'verifying': verifying.count(),
'send_status': {
'success': success.count(),
'unsent': unsent.count(),
'redirect': redirect.count(),
'client_error': client_error.count(),
'server_error': server_error.count(),
},
'send_status_rec': {
'success': success_rec.count(),
'unsent': unsent_rec.count(),
'redirect': redirect_rec.count(),
'client_error': client_error_rec.count(),
'server_error': server_error_rec.count(),
}
}
return report
def _verifier_audit_for_day(beginning, ending):
summary = {}
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'audit_period_ending': F('audit_period_beginning') + (60*60*24)
}
periodic_exists = models.InstanceExists.objects.filter(**filters)
summary['periodic'] = _audit_for_exists(periodic_exists)
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'audit_period_ending__lt': F('audit_period_beginning') + (60*60*24)
}
instant_exists = models.InstanceExists.objects.filter(**filters)
summary['instantaneous'] = _audit_for_exists(instant_exists)
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'status': models.InstanceExists.FAILED
}
failed = models.InstanceExists.objects.filter(**filters)
detail = []
for exist in failed:
detail.append(['Exist', exist.id, exist.fail_reason])
return summary, detail
def _launch_audit_for_period(beginning, ending):
launches_dict = {}
new_launches = _get_new_launches(beginning, ending)
@ -278,7 +186,6 @@ def _launch_audit_for_period(beginning, ending):
launch_to_exists_fails = _audit_launches_to_exists(launches_dict,
exists_dict,
beginning)
return launch_to_exists_fails, new_launches.count(), len(old_launches_dict)
@ -287,8 +194,9 @@ def audit_for_period(beginning, ending):
ending_decimal = dt.dt_to_decimal(ending)
(verify_summary,
verify_detail) = _verifier_audit_for_day(beginning_decimal,
ending_decimal)
verify_detail) = usage_audit._verifier_audit_for_day(beginning_decimal,
ending_decimal,
models.InstanceExists)
detail, new_count, old_count = _launch_audit_for_period(beginning_decimal,
ending_decimal)
@ -309,29 +217,6 @@ def audit_for_period(beginning, ending):
return summary, details
def get_previous_period(time, period_length):
if period_length == 'day':
last_period = time - datetime.timedelta(days=1)
start = datetime.datetime(year=last_period.year,
month=last_period.month,
day=last_period.day)
end = datetime.datetime(year=time.year,
month=time.month,
day=time.day)
return start, end
elif period_length == 'hour':
last_period = time - datetime.timedelta(hours=1)
start = datetime.datetime(year=last_period.year,
month=last_period.month,
day=last_period.day,
hour=last_period.hour)
end = datetime.datetime(year=time.year,
month=time.month,
day=time.day,
hour=time.hour)
return start, end
def store_results(start, end, summary, details):
values = {
'json': make_json_report(summary, details),
@ -393,7 +278,7 @@ if __name__ == '__main__':
else:
time = datetime.datetime.utcnow()
start, end = get_previous_period(time, args.period_length)
start, end = usage_audit.get_previous_period(time, args.period_length)
summary, details = audit_for_period(start, end)

107
reports/usage_audit.py Normal file
View File

@ -0,0 +1,107 @@
import datetime
from django.db.models import F
from stacktach import models
def _status_queries(exists_query):
verified = exists_query.filter(status=models.InstanceExists.VERIFIED)
reconciled = exists_query.filter(status=models.InstanceExists.RECONCILED)
fail = exists_query.filter(status=models.InstanceExists.FAILED)
pending = exists_query.filter(status=models.InstanceExists.PENDING)
verifying = exists_query.filter(status=models.InstanceExists.VERIFYING)
return verified, reconciled, fail, pending, verifying
def _send_status_queries(exists_query):
unsent = exists_query.filter(send_status=0)
success = exists_query.filter(send_status__gte=200,
send_status__lt=300)
redirect = exists_query.filter(send_status__gte=300,
send_status__lt=400)
client_error = exists_query.filter(send_status__gte=400,
send_status__lt=500)
server_error = exists_query.filter(send_status__gte=500,
send_status__lt=600)
return success, unsent, redirect, client_error, server_error
def _audit_for_exists(exists_query):
(verified, reconciled,
fail, pending, verifying) = _status_queries(exists_query)
(success, unsent, redirect,
client_error, server_error) = _send_status_queries(verified)
report = {
'count': exists_query.count(),
'verified': verified.count(),
'reconciled': reconciled.count(),
'failed': fail.count(),
'pending': pending.count(),
'verifying': verifying.count(),
'send_status': {
'success': success.count(),
'unsent': unsent.count(),
'redirect': redirect.count(),
'client_error': client_error.count(),
'server_error': server_error.count(),
}
}
return report
def _verifier_audit_for_day(beginning, ending, exists_model):
summary = {}
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'audit_period_ending': F('audit_period_beginning') + (60*60*24)
}
periodic_exists = exists_model.objects.filter(**filters)
summary['periodic'] = _audit_for_exists(periodic_exists)
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'audit_period_ending__lt': F('audit_period_beginning') + (60*60*24)
}
instant_exists = exists_model.objects.filter(**filters)
summary['instantaneous'] = _audit_for_exists(instant_exists)
filters = {
'raw__when__gte': beginning,
'raw__when__lte': ending,
'status': exists_model.FAILED
}
failed = exists_model.objects.filter(**filters)
detail = []
for exist in failed:
detail.append(['Exist', exist.id, exist.fail_reason])
return summary, detail
def get_previous_period(time, period_length):
if period_length == 'day':
last_period = time - datetime.timedelta(days=1)
start = datetime.datetime(year=last_period.year,
month=last_period.month,
day=last_period.day)
end = datetime.datetime(year=time.year,
month=time.month,
day=time.day)
return start, end
elif period_length == 'hour':
last_period = time - datetime.timedelta(hours=1)
start = datetime.datetime(year=last_period.year,
month=last_period.month,
day=last_period.day,
hour=last_period.hour)
end = datetime.datetime(year=time.year,
month=time.month,
day=time.day,
hour=time.hour)
return start, end