x/stacktach
Code Issues Proposed changes

Fixing merge conflicts

Browse Source
This commit is contained in:
Andrew Melton 2014-02-27 15:08:36 -05:00
commit 5ee5240339
8 changed files with 436 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/api.rst
View File

@ -805,4 +805,4 @@ stacky/usage/exists
]
]
:query instance: desired instance UUID (optional)
:query instance: desired instance UUID (optional)

73
docs/dbapi.rst
View File

@ -874,4 +874,75 @@ Returns a single instance exists matching provided id
"id": 5300,
"delete": null
}
}
}
db/count/verified/
==================
.. http:get:: http://example.com/count/verified/
Returns a count of .verified events stored in Stacktach's Rawdata table from
``audit_period_beginning`` to ``audit_period_ending``
**Query Parameters**
* ``audit_period_beginning``: datetime (yyyy-mm-dd)
* ``audit_period_ending``: datetime (yyyy-mm-dd)
* ``service``: ``nova`` or ``glance``. default="nova"
**Example request**:
.. sourcecode:: http
GET db/count/verified/ HTTP/1.1
Host: example.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: application/json
{
count: 10
}
repair
======
.. http:post:: http://example.com/repair/
Changes the status of all the exists of message-ids sent with the request
from 'pending' to 'sent_unverified' so that the verifier does not end up
sending .verified for all those exists(since the .exists have already been
modified as .verified and sent to AH by Yagi). It sends back the message-ids
of exists which could not be updated in the json response.
**Example request**:
.. sourcecode::http
POST /repair/ HTTP/1.1
Host: example.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/json
{
u'exists_not_pending': [u'494ebfce-0219-4b62-b810-79039a279620'],
u'absent_exists': [u'7609f3b2-3694-4b6f-869e-2f13ae504cb2',
u'0c64032e-4a60-44c0-a99d-5a4f2e46afb0']
}
:query message_ids: list of message_ids of exists messages
:query service: ``nova`` or ``glance``. default="nova"

48
stacktach/dbapi.py
View File

@ -21,6 +21,7 @@
import decimal
import functools
import json
from datetime import datetime
from django.db import transaction
from django.db.models import Count
@ -445,3 +446,50 @@ def _convert_model_list(model_list, extra_values_func=None):
converted.append(_convert_model(item, extra_values_func))
return converted
def _rawdata_factory(service):
if service == "nova":
rawdata = models.RawData.objects
elif service == "glance":
rawdata = models.GlanceRawData.objects
else:
raise BadRequestException(message="Invalid service")
return rawdata
@api_call
def get_verified_count(request):
try:
audit_period_beginning = datetime.strptime(
request.GET.get("audit_period_beginning"), "%Y-%m-%d")
audit_period_ending = datetime.strptime(
request.GET.get("audit_period_ending"), "%Y-%m-%d")
service = request.GET.get("service", "nova")
rawdata = _rawdata_factory(service)
filters = {
'when__gte': dt.dt_to_decimal(audit_period_beginning),
'when__lte': dt.dt_to_decimal(audit_period_ending),
'event': "compute.instance.exists.verified"
}
return {'count': rawdata.filter(**filters).count()}
except KeyError and TypeError:
raise BadRequestException(message="Invalid/absent query parameter")
except ValueError:
raise BadRequestException(message="Invalid format for date (Correct "
"format should be %YYYY-%mm-%dd)")
def repair_stacktach_down(request):
post_dict = dict((request.POST._iterlists()))
message_ids = post_dict.get('message_ids')
service = post_dict.get('service', ['nova'])
klass = _exists_model_factory(service[0])['klass']
absent_exists, exists_not_pending = \
klass.mark_exists_as_sent_unverified(message_ids)
response_data = {'absent_exists': absent_exists,
'exists_not_pending': exists_not_pending}
response = HttpResponse(json.dumps(response_data),
content_type="application/json")
return response

36
stacktach/models.py
View File

@ -338,6 +338,24 @@ class InstanceExists(models.Model):
def update_status(self, new_status):
self.status = new_status
@staticmethod
def mark_exists_as_sent_unverified(message_ids):
absent_exists = []
exists_not_pending = []
for message_id in message_ids:
try:
exists = InstanceExists.objects.get(message_id=message_id)
if exists.status == InstanceExists.PENDING:
exists.status = InstanceExists.SENT_UNVERIFIED
exists.save()
else:
exists_not_pending.append(message_id)
except Exception:
absent_exists.append(message_id)
return absent_exists, exists_not_pending
class Timing(models.Model):
"""Each Timing record corresponds to a .start/.end event pair
@ -536,6 +554,24 @@ class ImageExists(models.Model):
self.fail_reason = reason
self.save()
@staticmethod
def mark_exists_as_sent_unverified(message_ids):
absent_exists = []
exists_not_pending = []
for message_id in message_ids:
exists_list = ImageExists.objects.filter(message_id=message_id)
if exists_list:
for exists in exists_list:
if exists.status == ImageExists.PENDING:
exists.status = ImageExists.SENT_UNVERIFIED
exists.save()
else:
exists_not_pending.append(message_id)
else :
absent_exists.append(message_id)
return absent_exists, exists_not_pending
def get_model_fields(model):
return model._meta.fields

2
stacktach/urls.py
View File

@ -91,6 +91,8 @@ dbapi_urls = (
'stacktach.dbapi.get_usage_exist_stats'),
url(r'db/stats/glance/exists$',
'stacktach.dbapi.get_usage_exist_stats_glance'),
url(r'db/count/verified', 'stacktach.dbapi.get_verified_count'),
url(r'db/repair/', 'stacktach.dbapi.repair_stacktach_down'),
)
urlpatterns = patterns('', *(web_urls + stacky_urls + dbapi_urls))

151
tests/unit/test_dbapi.py
View File

@ -19,6 +19,7 @@
# IN THE SOFTWARE.
import datetime
from decimal import Decimal
import json
from django.db.models import Count
@ -129,8 +130,8 @@ class DBAPITestCase(StacktachBaseTestCase):
fake_request = self.mox.CreateMockAnything()
fake_request.GET = {'somebadfield_max': str(start_time)}
fake_model = self.make_fake_model()
fake_model._meta.get_field_by_name('somebadfield')\
.AndRaise(FieldDoesNotExist())
fake_model._meta.get_field_by_name('somebadfield') \
.AndRaise(FieldDoesNotExist())
self.mox.ReplayAll()
self.assertRaises(dbapi.BadRequestException, dbapi._get_filter_args,
@ -312,7 +313,8 @@ class DBAPITestCase(StacktachBaseTestCase):
fake_request.GET = filters
self.mox.StubOutWithMock(dbapi, '_get_filter_args')
dbapi._get_filter_args(fake_model, fake_request,
custom_filters=custom_filters).AndReturn(filters)
custom_filters=custom_filters).AndReturn(
filters)
self.mox.StubOutWithMock(dbapi, '_check_has_field')
dbapi._check_has_field(fake_model, 'id')
result = self.mox.CreateMockAnything()
@ -563,7 +565,8 @@ class DBAPITestCase(StacktachBaseTestCase):
exists1.send_status = 200
self.mox.VerifyAll()
def test_send_status_batch_accepts_post_for_nova_and_glance_when_version_is_1(self):
def test_send_status_batch_accepts_post_for_nova_and_glance_when_version_is_1(
self):
fake_request = self.mox.CreateMockAnything()
fake_request.method = 'POST'
fake_request.GET = {'service': 'glance'}
@ -591,14 +594,16 @@ class DBAPITestCase(StacktachBaseTestCase):
models.ImageExists.objects.select_for_update().AndReturn(results1)
exists1A = self.mox.CreateMockAnything()
exists1B = self.mox.CreateMockAnything()
results1.filter(message_id=MESSAGE_ID_2).AndReturn([exists1A, exists1B])
results1.filter(message_id=MESSAGE_ID_2).AndReturn(
[exists1A, exists1B])
exists1A.save()
exists1B.save()
results2 = self.mox.CreateMockAnything()
models.ImageExists.objects.select_for_update().AndReturn(results2)
exists2A = self.mox.CreateMockAnything()
exists2B = self.mox.CreateMockAnything()
results2.filter(message_id=MESSAGE_ID_1).AndReturn([exists2A, exists2B])
results2.filter(message_id=MESSAGE_ID_1).AndReturn(
[exists2A, exists2B])
exists2A.save()
exists2B.save()
trans_obj.__exit__(None, None, None)
@ -609,7 +614,6 @@ class DBAPITestCase(StacktachBaseTestCase):
self.mox.VerifyAll()
def test_send_status_batch_accepts_post_when_version_is_0(self):
fake_request = self.mox.CreateMockAnything()
fake_request.method = 'POST'
@ -764,7 +768,8 @@ class DBAPITestCase(StacktachBaseTestCase):
launches = {'a': 1}
self.mox.StubOutWithMock(dbapi, '_convert_model_list')
dbapi._convert_model_list(mock_objects).AndReturn(launches)
dbapi.get_db_objects(models.InstanceUsage, fake_request, 'launched_at').AndReturn(mock_objects)
dbapi.get_db_objects(models.InstanceUsage, fake_request,
'launched_at').AndReturn(mock_objects)
self.mox.ReplayAll()
resp = dbapi.list_usage_launches(fake_request)
@ -781,7 +786,8 @@ class DBAPITestCase(StacktachBaseTestCase):
launches = {'a': 1}
self.mox.StubOutWithMock(dbapi, '_convert_model_list')
dbapi._convert_model_list(mock_objects).AndReturn(launches)
dbapi.get_db_objects(models.ImageUsage, fake_request, 'created_at').AndReturn(mock_objects)
dbapi.get_db_objects(models.ImageUsage, fake_request,
'created_at').AndReturn(mock_objects)
self.mox.ReplayAll()
resp = dbapi.list_usage_images(fake_request)
@ -798,7 +804,8 @@ class DBAPITestCase(StacktachBaseTestCase):
launches = {'a': 1}
self.mox.StubOutWithMock(dbapi, '_convert_model_list')
dbapi._convert_model_list(mock_objects).AndReturn(launches)
dbapi.get_db_objects(models.InstanceUsage, fake_request, 'launched_at').AndReturn(mock_objects)
dbapi.get_db_objects(models.InstanceUsage, fake_request,
'launched_at').AndReturn(mock_objects)
self.mox.ReplayAll()
resp = dbapi.list_usage_launches(fake_request)
@ -885,7 +892,8 @@ class DBAPITestCase(StacktachBaseTestCase):
deletes = {'a': 1}
self.mox.StubOutWithMock(dbapi, '_convert_model_list')
dbapi._convert_model_list(mock_objects).AndReturn(deletes)
dbapi.get_db_objects(models.InstanceDeletes, fake_request, 'launched_at').AndReturn(mock_objects)
dbapi.get_db_objects(models.InstanceDeletes, fake_request,
'launched_at').AndReturn(mock_objects)
self.mox.ReplayAll()
resp = dbapi.list_usage_deletes(fake_request)
@ -902,7 +910,8 @@ class DBAPITestCase(StacktachBaseTestCase):
deletes = {'a': 1}
self.mox.StubOutWithMock(dbapi, '_convert_model_list')
dbapi._convert_model_list(mock_objects).AndReturn(deletes)
dbapi.get_db_objects(models.InstanceDeletes, fake_request, 'launched_at').AndReturn(mock_objects)
dbapi.get_db_objects(models.InstanceDeletes, fake_request,
'launched_at').AndReturn(mock_objects)
self.mox.ReplayAll()
resp = dbapi.list_usage_deletes(fake_request)
@ -919,7 +928,8 @@ class DBAPITestCase(StacktachBaseTestCase):
deletes = {'a': 1}
self.mox.StubOutWithMock(dbapi, '_convert_model_list')
dbapi._convert_model_list(mock_objects).AndReturn(deletes)
dbapi.get_db_objects(models.ImageDeletes, fake_request, 'deleted_at').AndReturn(mock_objects)
dbapi.get_db_objects(models.ImageDeletes, fake_request,
'deleted_at').AndReturn(mock_objects)
self.mox.ReplayAll()
resp = dbapi.list_usage_deletes_glance(fake_request)
@ -1092,3 +1102,118 @@ class DBAPITestCase(StacktachBaseTestCase):
expected_response = json.dumps({'stats': result})
self.assertEqual(expected_response, response.content)
self.mox.VerifyAll()
def test_get_verified_count(self):
fake_request = self.mox.CreateMockAnything()
fake_request.method = 'GET'
fake_request.GET = {'audit_period_beginning': "2014-02-26",
'audit_period_ending': "2014-02-27",
'service': "nova"}
mock_query = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(models.RawData.objects, "filter")
models.RawData.objects.filter(event='compute.instance.exists.verified',
when__gte=Decimal('1393372800'),
when__lte=Decimal('1393459200')).\
AndReturn(mock_query)
mock_query.count().AndReturn(100)
self.mox.ReplayAll()
response = dbapi.get_verified_count(fake_request)
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.content), {'count': 100})
self.mox.VerifyAll()
def test_get_verified_count_wrong_date_format_returns_400(self):
fake_request = self.mox.CreateMockAnything()
fake_request.method = 'GET'
fake_request.GET = {'audit_period_beginning': "2014-020-26",
'service': "nova"}
self.mox.ReplayAll()
response = dbapi.get_verified_count(fake_request)
self.assertEqual(response.status_code, 400)
self.assertEqual(json.loads(response.content)['message'],
"Invalid format for date"
" (Correct format should be %YYYY-%mm-%dd)")
self.mox.VerifyAll()
def test_get_verified_count_wrong_service_returns_400(self):
fake_request = self.mox.CreateMockAnything()
fake_request.method = 'GET'
fake_request.GET = {'audit_period_beginning': "2014-02-26",
"audit_period_ending": "2014-02-27",
'service': "qonos"}
self.mox.ReplayAll()
response = dbapi.get_verified_count(fake_request)
self.assertEqual(response.status_code, 400)
self.assertEqual(json.loads(response.content)['message'],
"Invalid service")
self.mox.VerifyAll()
def test_get_verified_count_invalid_query_parameter_returns_400(self):
fake_request = self.mox.CreateMockAnything()
fake_request.method = 'GET'
fake_request.GET = {'audit_period': "2014-02-26",}
self.mox.ReplayAll()
response = dbapi.get_verified_count(fake_request)
self.assertEqual(response.status_code, 400)
self.assertEqual(json.loads(response.content)['message'],
"Invalid/absent query parameter")
self.mox.VerifyAll()
class StacktachRepairScenarioApi(StacktachBaseTestCase):
def setUp(self):
self.mox = mox.Mox()
def tearDown(self):
self.mox.UnsetStubs()
def test_change_nova_exists_status_for_all_exists(self):
request = self.mox.CreateMockAnything()
request.POST = self.mox.CreateMockAnything()
message_ids = ["04fd94b5-64dd-4559-83b7-981d9d4f7a5a",
"14fd94b5-64dd-4559-83b7-981d9d4f7a5a",
"24fd94b5-64dd-4559-83b7-981d9d4f7a5a"]
request.POST._iterlists().AndReturn([('service', ['nova']),
('message_ids', message_ids)])
self.mox.StubOutWithMock(models.InstanceExists,
'mark_exists_as_sent_unverified')
models.InstanceExists.mark_exists_as_sent_unverified(message_ids).\
AndReturn([[], []])
self.mox.ReplayAll()
response = dbapi.repair_stacktach_down(request)
self.assertEqual(response.status_code, 200)
response_data = json.loads(response.content)
self.assertEqual(response_data['exists_not_pending'], [])
self.assertEqual(response_data['absent_exists'], [])
self.mox.VerifyAll()
def test_change_glance_exists_status_for_all_exists(self):
request = self.mox.CreateMockAnything()
request.POST = self.mox.CreateMockAnything()
message_ids = ['04fd94b5-64dd-4559-83b7-981d9d4f7a5a',
'14fd94b5-64dd-4559-83b7-981d9d4f7a5a',
'24fd94b5-64dd-4559-83b7-981d9d4f7a5a']
request.POST._iterlists().AndReturn([('service', ['glance']),
('message_ids', message_ids)])
self.mox.StubOutWithMock(models.ImageExists,
'mark_exists_as_sent_unverified')
models.ImageExists.mark_exists_as_sent_unverified(message_ids).\
AndReturn([[], []])
self.mox.ReplayAll()
response = dbapi.repair_stacktach_down(request)
self.assertEqual(response.status_code, 200)
response_data = json.loads(response.content)
self.assertEqual(response_data['exists_not_pending'], [])
self.assertEqual(response_data['absent_exists'], [])
self.mox.VerifyAll()

138
tests/unit/test_models.py
View File

@ -112,6 +112,81 @@ class ImageExistsTestCase(unittest.TestCase):
'owner1-3': [exist4],
'owner2-2': [exist2]})
def test_mark_exists_as_sent_unverified(self):
message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b",
"9156b83e-f684-4ec3-8f94-7e41902f27aa"]
exist1 = self.mox.CreateMockAnything()
exist1.status = "pending"
exist1.save()
exist2 = self.mox.CreateMockAnything()
exist2.status = "pending"
exist2.save()
exist3 = self.mox.CreateMockAnything()
exist3.status = "pending"
exist3.save()
self.mox.StubOutWithMock(ImageExists.objects, 'filter')
ImageExists.objects.filter(message_id=message_ids[0]).AndReturn(
[exist1, exist2])
ImageExists.objects.filter(message_id=message_ids[1]).AndReturn(
[exist3])
self.mox.ReplayAll()
results = ImageExists.mark_exists_as_sent_unverified(message_ids)
self.assertEqual(results, ([], []))
self.mox.VerifyAll()
def test_mark_exists_as_sent_unverified_return_absent_exists(self):
message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b",
"9156b83e-f684-4ec3-8f94-7e41902f27aa"]
exist1 = self.mox.CreateMockAnything()
exist1.status = "pending"
exist1.save()
exist2 = self.mox.CreateMockAnything()
exist2.status = "pending"
exist2.save()
self.mox.StubOutWithMock(ImageExists.objects, 'filter')
ImageExists.objects.filter(message_id=message_ids[0]).AndReturn(
[exist1, exist2])
ImageExists.objects.filter(message_id=message_ids[1]).AndReturn([])
self.mox.ReplayAll()
results = ImageExists.mark_exists_as_sent_unverified(message_ids)
self.assertEqual(results, (['9156b83e-f684-4ec3-8f94-7e41902f27aa'],
[]))
self.mox.VerifyAll()
def test_mark_exists_as_sent_unverified_and_return_exist_not_pending(self):
message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b",
"9156b83e-f684-4ec3-8f94-7e41902f27aa"]
exist1 = self.mox.CreateMockAnything()
exist1.status = "pending"
exist1.save()
exist2 = self.mox.CreateMockAnything()
exist2.status = "verified"
exist3 = self.mox.CreateMockAnything()
exist3.status = "pending"
exist3.save()
self.mox.StubOutWithMock(ImageExists.objects, 'filter')
ImageExists.objects.filter(message_id=message_ids[0]).AndReturn(
[exist1, exist2])
ImageExists.objects.filter(message_id=message_ids[1]).AndReturn(
[exist3])
self.mox.ReplayAll()
results = ImageExists.mark_exists_as_sent_unverified(message_ids)
self.assertEqual(results, ([],
["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b"]))
self.mox.VerifyAll()
class InstanceExistsTestCase(unittest.TestCase):
def setUp(self):
@ -137,3 +212,66 @@ class InstanceExistsTestCase(unittest.TestCase):
self.mox.VerifyAll()
self.assertEqual(results, [1, 2])
def test_mark_exists_as_sent_unverified(self):
message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b",
"9156b83e-f684-4ec3-8f94-7e41902f27aa"]
exist1 = self.mox.CreateMockAnything()
exist1.status = "pending"
exist1.save()
exist2 = self.mox.CreateMockAnything()
exist2.status = "pending"
exist2.save()
self.mox.StubOutWithMock(InstanceExists.objects, 'get')
InstanceExists.objects.get(message_id=message_ids[0]).AndReturn(exist1)
InstanceExists.objects.get(message_id=message_ids[1]).AndReturn(exist2)
self.mox.ReplayAll()
results = InstanceExists.mark_exists_as_sent_unverified(message_ids)
self.assertEqual(results, ([], []))
self.mox.VerifyAll()
def test_mark_exists_as_sent_unverified_return_absent_exists(self):
message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b",
"9156b83e-f684-4ec3-8f94-7e41902f27aa"]
exist1 = self.mox.CreateMockAnything()
exist1.status = "pending"
exist1.save()
self.mox.StubOutWithMock(InstanceExists.objects, 'get')
InstanceExists.objects.get(message_id=message_ids[0]).AndReturn(exist1)
InstanceExists.objects.get(message_id=message_ids[1]).AndRaise(
Exception)
self.mox.ReplayAll()
results = InstanceExists.mark_exists_as_sent_unverified(message_ids)
self.assertEqual(results, (['9156b83e-f684-4ec3-8f94-7e41902f27aa'],
[]))
self.mox.VerifyAll()
def test_mark_exists_as_sent_unverified_and_return_exist_not_pending(self):
message_ids = ["0708cb0b-6169-4d7c-9f58-3cf3d5bf694b",
"9156b83e-f684-4ec3-8f94-7e41902f27aa"]
exist1 = self.mox.CreateMockAnything()
exist1.status = "pending"
exist1.save()
exist2 = self.mox.CreateMockAnything()
exist2.status = "verified"
self.mox.StubOutWithMock(InstanceExists.objects, 'get')
InstanceExists.objects.get(message_id=message_ids[0]).AndReturn(exist1)
InstanceExists.objects.get(message_id=message_ids[1]).AndReturn(exist2)
self.mox.ReplayAll()
results = InstanceExists.mark_exists_as_sent_unverified(message_ids)
self.assertEqual(results, ([],
["9156b83e-f684-4ec3-8f94-7e41902f27aa"]))
self.mox.VerifyAll()

3
tests/unit/test_stacktach.py
View File

@ -38,8 +38,7 @@ from utils import TENANT_ID_1
from utils import INSTANCE_TYPE_ID_1
from utils import DUMMY_TIME
from utils import INSTANCE_TYPE_ID_2
from utils import IMAGE_UUID_1
from stacktach import stacklog
from stacktach import stacklog, models
from stacktach import notification
from stacktach import views
from tests.unit import StacktachBaseTestCase