From 065a2d76f280f5676ef24dd082c071825e0ade47 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Mon, 19 Aug 2013 15:34:59 +0530 Subject: [PATCH 01/16] added null/type checks on specific fields of exit --- stacktach/models.py | 8 +- tests/unit/test_glance_verifier.py | 149 ++++++++++++++++++- tests/unit/test_nova_verifier.py | 226 +++++++++++++++++++++++++++++ verifier/__init__.py | 12 ++ verifier/base_verifier.py | 29 ++++ verifier/glance_verifier.py | 19 ++- verifier/nova_verifier.py | 20 +++ 7 files changed, 457 insertions(+), 6 deletions(-) diff --git a/stacktach/models.py b/stacktach/models.py index a34f494..658c959 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -424,7 +424,7 @@ class ImageUsage(models.Model): uuid = models.CharField(max_length=50, db_index=True) created_at = models.DecimalField(max_digits=20, decimal_places=6, db_index=True) - owner = models.CharField(max_length=50, db_index=True) + owner = models.CharField(max_length=50, db_index=True, null=True, blank=True) size = models.BigIntegerField(max_length=20) last_raw = models.ForeignKey(GlanceRawData, null=True) @@ -456,7 +456,7 @@ class ImageExists(models.Model): (FAILED, 'Failed Verification'), ] - uuid = models.CharField(max_length=50, db_index=True) + uuid = models.CharField(max_length=50, db_index=True, null=True) created_at = models.DecimalField(max_digits=20, decimal_places=6, db_index=True, null=True) @@ -476,8 +476,8 @@ class ImageExists(models.Model): usage = models.ForeignKey(ImageUsage, related_name='+', null=True) delete = models.ForeignKey(ImageDeletes, related_name='+', null=True) send_status = models.IntegerField(default=0, db_index=True) - owner = models.CharField(max_length=255, db_index=True) - size = models.BigIntegerField(max_length=20) + owner = models.CharField(max_length=255, db_index=True, null=True, blank=True) + size = models.BigIntegerField(max_length=20, null=True) def update_status(self, new_status): self.status = new_status diff --git a/tests/unit/test_glance_verifier.py b/tests/unit/test_glance_verifier.py index 5a8b726..f96b375 100644 --- a/tests/unit/test_glance_verifier.py +++ b/tests/unit/test_glance_verifier.py @@ -32,6 +32,8 @@ from tests.unit import StacktachBaseTestCase from utils import IMAGE_UUID_1 from utils import make_verifier_config from verifier import glance_verifier +from verifier import NullFieldException +from verifier import WrongTypeException from verifier import FieldMismatch from verifier import NotFound from verifier import VerificationException @@ -261,12 +263,156 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): self.assertEqual(fm.actual, decimal.Decimal('6.1')) self.mox.VerifyAll() + def test_should_verify_that_image_size_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = None + exist.created_at = decimal.Decimal('5.1') + exist.uuid = 'abcd1234' + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'image_size') + self.assertEqual(nf.reason, "image_size field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_created_at_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = None + exist.uuid = 'abcd1234' + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'created_at') + self.assertEqual(nf.reason, "created_at field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_uuid_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = decimal.Decimal('5.1') + exist.uuid = None + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'uuid') + self.assertEqual(nf.reason, "uuid field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_owner_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 1234 + exist.created_at = decimal.Decimal('5.1') + exist.uuid = 'abcd1234' + exist.owner = None + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'owner') + self.assertEqual(nf.reason, "owner field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_uuid_value_is_uuid_like(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "asdfe-fgh" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'uuid') + self.assertEqual(wt.reason, "{ uuid : asdfe-fgh } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_created_at_is_decimal(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = "123.a" + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'created_at') + self.assertEqual(wt.reason, "{ created_at : 123.a } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_image_size_is_of_type_decimal(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 'size' + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'size') + self.assertEqual(wt.reason, "{ size : size } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_owner_is_of_type_hex(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 1234 + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + exist.owner = "3762854cd6f6435998188d5120e4c271,kl" + self.mox.ReplayAll() + + try: + glance_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'owner') + self.assertEqual(wt.reason, "{ owner : 3762854cd6f6435998188d5120e4c271,kl } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_correctly_for_all_non_null_and_valid_types(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + exist.size = 983040 + exist.created_at = decimal.Decimal('5.1') + exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" + exist.owner = "3762854cd6f6435998188d5120e4c271" + self.mox.ReplayAll() + + glance_verifier._verify_validity(exist) + self.mox.VerifyAll() + def test_verify_should_verify_exists_for_usage_and_delete(self): exist = self.mox.CreateMockAnything() self.mox.StubOutWithMock(glance_verifier, '_verify_for_usage') glance_verifier._verify_for_usage(exist) self.mox.StubOutWithMock(glance_verifier, '_verify_for_delete') glance_verifier._verify_for_delete(exist) + self.mox.StubOutWithMock(glance_verifier, '_verify_validity') + glance_verifier._verify_validity(exist) exist.mark_verified() self.mox.ReplayAll() @@ -413,4 +559,5 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): self.glance_verifier.send_verified_notification(exist, exchange, connection) - self.mox.VerifyAll() \ No newline at end of file + self.mox.VerifyAll() + diff --git a/tests/unit/test_nova_verifier.py b/tests/unit/test_nova_verifier.py index a5452e1..b6b080b 100644 --- a/tests/unit/test_nova_verifier.py +++ b/tests/unit/test_nova_verifier.py @@ -45,6 +45,8 @@ from utils import TENANT_ID_1 from utils import TENANT_ID_2 from utils import INSTANCE_TYPE_ID_1 from verifier import nova_verifier +from verifier import NullFieldException +from verifier import WrongTypeException from verifier import AmbiguousResults from verifier import FieldMismatch from verifier import NotFound @@ -324,6 +326,228 @@ class NovaVerifierTestCase(StacktachBaseTestCase): self.mox.VerifyAll() + def test_should_verify_that_tenant_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.tenant = None + exist.id = 23 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'tenant') + self.assertEqual(nf.reason, "tenant field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_launched_at_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.tenant = 'tenant' + exist.id = 23 + exist.launched_at = None + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'launched_at') + self.assertEqual(nf.reason, "launched_at field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_that_instance_type_id_in_exist_is_not_null(self): + exist = self.mox.CreateMockAnything() + exist.tenant = 'tenant' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = None + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except NullFieldException as nf: + self.assertEqual(nf.field_name, 'instance_type_id') + self.assertEqual(nf.reason, "instance_type_id field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_tenant_id_is_of_type_hex(self): + exist = self.mox.CreateMockAnything() + exist.tenant = 'tenant' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = 2 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'tenant') + self.assertEqual(wt.reason, "{ tenant : tenant } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_flavor_is_of_type_integer(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = 'flavor' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'instance_type_id') + self.assertEqual(wt.reason, "{ instance_type_id : flavor } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_launched_at_is_of_type_decimal(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = 111 + exist.instance_type_id = 4 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'launched_at') + self.assertEqual(wt.reason, "{ launched_at : 111 } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_deleted_at_is_of_decimal_type_if_present(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.instance_type_id = 4 + exist.deleted_at = 20 + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'deleted_at') + self.assertEqual(wt.reason, "{ deleted_at : 20 } of incorrect type for exist id 23") + self.mox.VerifyAll() + + + def test_should_verify_rax_options_should_be_of_integer_type(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 'a' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'rax_options') + self.assertEqual(wt.reason, "{ rax_options : a } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_arch_should_be_alphanumeric(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64,' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'os_arch') + self.assertEqual(wt.reason, "{ os_arch : x64, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_distro_should_be_alphanumeric(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server,' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'os_distro') + self.assertEqual(wt.reason, "{ os_distro : com.microsoft.server, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_version_should_be_alphanumeric(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server' + exist.os_version = '2008.2,' + self.mox.ReplayAll() + + try: + nova_verifier._verify_validity(exist) + self.fail() + except WrongTypeException as wt: + self.assertEqual(wt.field_name, 'os_version') + self.assertEqual(wt.reason, "{ os_version : 2008.2, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_exist_fields_correctly(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server' + exist.os_version = '2008.2' + self.mox.ReplayAll() + + nova_verifier._verify_validity(exist) + self.mox.VerifyAll() + + def test_should_verify_exist_fields_correctly_and_deleted_at_can_be_none(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = None + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_arch = 'x64' + exist.os_distro = 'com.microsoft.server' + exist.os_version = '2008.2' + self.mox.ReplayAll() + + nova_verifier._verify_validity(exist) + self.mox.VerifyAll() + def test_verify_for_delete(self): exist = self.mox.CreateMockAnything() exist.delete = self.mox.CreateMockAnything() @@ -571,9 +795,11 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.launched_at = decimal.Decimal('1.1') self.mox.StubOutWithMock(nova_verifier, '_verify_for_launch') self.mox.StubOutWithMock(nova_verifier, '_verify_for_delete') + self.mox.StubOutWithMock(nova_verifier, '_verify_validity') self.mox.StubOutWithMock(exist, 'mark_verified') nova_verifier._verify_for_launch(exist) nova_verifier._verify_for_delete(exist) + nova_verifier._verify_validity(exist) exist.mark_verified() self.mox.ReplayAll() result, exists = nova_verifier._verify(exist) diff --git a/verifier/__init__.py b/verifier/__init__.py index eb30360..0c542f6 100644 --- a/verifier/__init__.py +++ b/verifier/__init__.py @@ -51,3 +51,15 @@ class FieldMismatch(VerificationException): self.reason = "Expected %s to be '%s' got '%s'" % (self.field_name, self.expected, self.actual) + +class NullFieldException(VerificationException): + def __init__(self, field_name, exist_id): + self.field_name = field_name + self.reason = "%s field was null for exist id %s" %(field_name, exist_id) + +class WrongTypeException(VerificationException): + def __init__(self, field_name, value, exist_id): + self.field_name = field_name + self.reason = "{ %s : %s } of incorrect type for exist id %s"\ + %(field_name, value, exist_id) + diff --git a/verifier/base_verifier.py b/verifier/base_verifier.py index cea72b7..01f9209 100644 --- a/verifier/base_verifier.py +++ b/verifier/base_verifier.py @@ -19,7 +19,9 @@ # IN THE SOFTWARE. import datetime +import decimal import os +import re import sys import time import multiprocessing @@ -32,6 +34,8 @@ POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) +from verifier import WrongTypeException + from stacktach import stacklog, message_service LOG = stacklog.get_logger('verifier') @@ -65,6 +69,31 @@ def _verify_date_field(d1, d2, same_second=False): return False +def _is_like_uuid(attr_name, attr_value, exist_id): + if not re.match("[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$", + attr_value): + raise WrongTypeException(attr_name, attr_value, exist_id) + + +def _is_like_date(attr_name, attr_value, exist_id): + if not isinstance(attr_value, decimal.Decimal): + raise WrongTypeException(attr_name, attr_value, exist_id) + + +def _is_int(attr_name, attr_value, exist_id): + if not isinstance(attr_value, int): + raise WrongTypeException(attr_name, attr_value, exist_id) + + +def _is_hex_owner_id(attr_name, attr_value, exist_id): + if not re.match("[0-9a-f]{32}$", attr_value): + raise WrongTypeException(attr_name, attr_value, exist_id) + +def _is_alphanumeric(attr_name, attr_value, exist_id): + if not re.match("[a-zA-Z0-9.]+$", attr_value): + raise WrongTypeException(attr_name, attr_value, exist_id) + + class Verifier(object): def __init__(self, config, pool=None, reconciler=None): self.config = config diff --git a/verifier/glance_verifier.py b/verifier/glance_verifier.py index 3947cfe..0ba611c 100644 --- a/verifier/glance_verifier.py +++ b/verifier/glance_verifier.py @@ -31,7 +31,10 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) from stacktach import models -from verifier import FieldMismatch, VerificationException, base_verifier +from verifier import FieldMismatch +from verifier import VerificationException +from verifier import base_verifier +from verifier import NullFieldException from verifier import NotFound from stacktach import datetime_to_decimal as dt import datetime @@ -54,6 +57,19 @@ def _verify_field_mismatch(exists, usage): usage.size) +def _verify_validity(exist): + fields = {exist.size: 'image_size', exist.created_at: 'created_at', + exist.uuid: 'uuid', exist.owner: 'owner'} + for (field_value, field_name) in fields.items(): + if field_value is None: + raise NullFieldException(field_name, exist.id) + base_verifier._is_like_uuid('uuid', exist.uuid, exist.id) + base_verifier._is_like_date('created_at', exist.created_at, exist.id) + base_verifier._is_int('size', exist.size, exist.id) + base_verifier._is_hex_owner_id('owner', exist.owner, exist.id) + + + def _verify_for_usage(exist, usage=None): usage_type = "ImageUsage" if not usage and exist.usage: @@ -116,6 +132,7 @@ def _verify(exist): try: _verify_for_usage(exist) _verify_for_delete(exist) + _verify_validity(exist) verified = True exist.mark_verified() diff --git a/verifier/nova_verifier.py b/verifier/nova_verifier.py index 605d232..5a65e2f 100644 --- a/verifier/nova_verifier.py +++ b/verifier/nova_verifier.py @@ -32,6 +32,7 @@ if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')): sys.path.insert(0, POSSIBLE_TOPDIR) from verifier import base_verifier +from verifier import NullFieldException from stacktach import models from stacktach import datetime_to_decimal as dt from verifier import FieldMismatch @@ -145,6 +146,24 @@ def _verify_for_delete(exist, delete=None, 'deleted_at', exist.deleted_at, delete.deleted_at) +def _verify_validity(exist): + fields = {exist.tenant: 'tenant', exist.launched_at: 'launched_at', + exist.instance_type_id: 'instance_type_id'} + for (field_value, field_name) in fields.items(): + if field_value is None: + raise NullFieldException(field_name, exist.id) + base_verifier._is_hex_owner_id('tenant', exist.tenant, exist.id) + base_verifier._is_int('instance_type_id', exist.instance_type_id, exist.id) + base_verifier._is_like_date('launched_at', exist.launched_at, exist.id) + if exist.deleted_at is not None: + base_verifier._is_like_date('deleted_at', exist.deleted_at, exist.id) + base_verifier._is_int('rax_options', exist.rax_options, exist.id) + base_verifier._is_alphanumeric('os_arch', exist.os_arch, exist.id) + base_verifier._is_alphanumeric('os_distro', exist.os_distro, exist.id) + base_verifier._is_alphanumeric('os_version', exist.os_version, exist.id) + + + def _verify_with_reconciled_data(exist): if not exist.launched_at: raise VerificationException("Exists without a launched_at") @@ -201,6 +220,7 @@ def _verify(exist): _verify_for_launch(exist) _verify_for_delete(exist) + _verify_validity(exist) verified = True exist.mark_verified() From e294145f66290371dc6f39233d956a872a465c96 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Mon, 26 Aug 2013 12:09:55 +0530 Subject: [PATCH 02/16] Fixed type check for instance_type_id and rax_options for instanceexists and size for glanceexists changed the order of the validity check added validation level as flag Added empty check for nova image properties --- etc/sample_stacktach_verifier_config.json | 1 + tests/unit/test_glance_verifier.py | 4 +- tests/unit/test_nova_verifier.py | 339 ++++++++++++++-------- tests/unit/utils.py | 1 + verifier/base_verifier.py | 12 +- verifier/config.py | 4 + verifier/glance_verifier.py | 3 +- verifier/nova_verifier.py | 46 ++- 8 files changed, 277 insertions(+), 133 deletions(-) diff --git a/etc/sample_stacktach_verifier_config.json b/etc/sample_stacktach_verifier_config.json index 54b67fa..f18e901 100644 --- a/etc/sample_stacktach_verifier_config.json +++ b/etc/sample_stacktach_verifier_config.json @@ -4,6 +4,7 @@ "settle_units": "minutes", "pool_size": 2, "enable_notifications": true, + "validation_level": "all", "rabbit": { "durable_queue": false, "host": "10.0.0.1", diff --git a/tests/unit/test_glance_verifier.py b/tests/unit/test_glance_verifier.py index f96b375..0bc4e1f 100644 --- a/tests/unit/test_glance_verifier.py +++ b/tests/unit/test_glance_verifier.py @@ -379,7 +379,7 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): def test_should_verify_owner_is_of_type_hex(self): exist = self.mox.CreateMockAnything() exist.id = 23 - exist.size = 1234 + exist.size = 1234L exist.created_at = decimal.Decimal('5.1') exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" exist.owner = "3762854cd6f6435998188d5120e4c271,kl" @@ -396,7 +396,7 @@ class GlanceVerifierTestCase(StacktachBaseTestCase): def test_should_verify_correctly_for_all_non_null_and_valid_types(self): exist = self.mox.CreateMockAnything() exist.id = 23 - exist.size = 983040 + exist.size = 983040L exist.created_at = decimal.Decimal('5.1') exist.uuid = "58fb036d-5ef8-47a8-b503-7571276c400a" exist.owner = "3762854cd6f6435998188d5120e4c271" diff --git a/tests/unit/test_nova_verifier.py b/tests/unit/test_nova_verifier.py index b6b080b..63f2506 100644 --- a/tests/unit/test_nova_verifier.py +++ b/tests/unit/test_nova_verifier.py @@ -290,12 +290,11 @@ class NovaVerifierTestCase(StacktachBaseTestCase): results.count().AndReturn(0) self.mox.ReplayAll() - try: + with self.assertRaises(NotFound) as nf: nova_verifier._verify_for_launch(exist) - self.fail() - except NotFound, nf: - self.assertEqual(nf.object_type, 'InstanceUsage') - self.assertEqual(nf.search_params, {'instance': INSTANCE_ID_1}) + exception = nf.exception + self.assertEqual(exception.object_type, 'InstanceUsage') + self.assertEqual(exception.search_params, {'instance': INSTANCE_ID_1}) self.mox.VerifyAll() @@ -315,14 +314,13 @@ class NovaVerifierTestCase(StacktachBaseTestCase): results.count().AndReturn(2) self.mox.ReplayAll() - try: + with self.assertRaises(AmbiguousResults) as ar: nova_verifier._verify_for_launch(exist) - self.fail() - except AmbiguousResults, nf: - self.assertEqual(nf.object_type, 'InstanceUsage') - search_params = {'instance': INSTANCE_ID_1, - 'launched_at': decimal.Decimal('1.1')} - self.assertEqual(nf.search_params, search_params) + exception = ar.exception + self.assertEqual(exception.object_type, 'InstanceUsage') + search_params = {'instance': INSTANCE_ID_1, + 'launched_at': decimal.Decimal('1.1')} + self.assertEqual(exception.search_params, search_params) self.mox.VerifyAll() @@ -332,12 +330,12 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.id = 23 self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except NullFieldException as nf: - self.assertEqual(nf.field_name, 'tenant') - self.assertEqual(nf.reason, "tenant field was null for exist id 23") + with self.assertRaises(NullFieldException) as nf: + nova_verifier._verify_validity(exist, 'all') + exception = nf.exception + self.assertEqual(exception.field_name, 'tenant') + self.assertEqual(exception.reason, + "tenant field was null for exist id 23") self.mox.VerifyAll() def test_should_verify_that_launched_at_in_exist_is_not_null(self): @@ -347,12 +345,12 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.launched_at = None self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except NullFieldException as nf: - self.assertEqual(nf.field_name, 'launched_at') - self.assertEqual(nf.reason, "launched_at field was null for exist id 23") + with self.assertRaises(NullFieldException) as nf: + nova_verifier._verify_validity(exist, 'all') + exception = nf.exception + self.assertEqual(exception.field_name, 'launched_at') + self.assertEqual(exception.reason, + "launched_at field was null for exist id 23") self.mox.VerifyAll() def test_should_verify_that_instance_type_id_in_exist_is_not_null(self): @@ -363,12 +361,12 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.instance_type_id = None self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except NullFieldException as nf: - self.assertEqual(nf.field_name, 'instance_type_id') - self.assertEqual(nf.reason, "instance_type_id field was null for exist id 23") + with self.assertRaises(NullFieldException) as nf: + nova_verifier._verify_validity(exist, 'all') + exception = nf.exception + self.assertEqual(exception.field_name, 'instance_type_id') + self.assertEqual(exception.reason, + "instance_type_id field was null for exist id 23") self.mox.VerifyAll() def test_should_verify_tenant_id_is_of_type_hex(self): @@ -379,12 +377,13 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.instance_type_id = 2 self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'tenant') - self.assertEqual(wt.reason, "{ tenant : tenant } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'tenant') + self.assertEqual( + exception.reason, + "{ tenant : tenant } of incorrect type for exist id 23") self.mox.VerifyAll() def test_should_verify_flavor_is_of_type_integer(self): @@ -395,12 +394,13 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.instance_type_id = 'flavor' self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'instance_type_id') - self.assertEqual(wt.reason, "{ instance_type_id : flavor } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'instance_type_id') + self.assertEqual( + exception.reason, + "{ instance_type_id : flavor } of incorrect type for exist id 23") self.mox.VerifyAll() def test_should_verify_launched_at_is_of_type_decimal(self): @@ -411,12 +411,13 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.instance_type_id = 4 self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'launched_at') - self.assertEqual(wt.reason, "{ launched_at : 111 } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'launched_at') + self.assertEqual( + exception.reason, + "{ launched_at : 111 } of incorrect type for exist id 23") self.mox.VerifyAll() def test_should_verify_deleted_at_is_of_decimal_type_if_present(self): @@ -428,12 +429,13 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.deleted_at = 20 self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'deleted_at') - self.assertEqual(wt.reason, "{ deleted_at : 20 } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'deleted_at') + self.assertEqual( + exception.reason, + "{ deleted_at : 20 } of incorrect type for exist id 23") self.mox.VerifyAll() @@ -447,12 +449,30 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.rax_options = 'a' self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'rax_options') - self.assertEqual(wt.reason, "{ rax_options : a } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'rax_options') + self.assertEqual( + exception.reason, + "{ rax_options : a } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_rax_options_should_not_be_empty(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = '' + self.mox.ReplayAll() + + with self.assertRaises(NullFieldException) as nf: + nova_verifier._verify_validity(exist, 'all') + exception = nf.exception + self.assertEqual(exception.field_name, 'rax_options') + self.assertEqual(exception.reason, "rax_options field was null for exist id 23") self.mox.VerifyAll() def test_should_verify_os_arch_should_be_alphanumeric(self): @@ -464,15 +484,37 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.instance_type_id = 4 exist.rax_options = 12 - exist.os_arch = 'x64,' + exist.os_architecture = 'x64,' self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'os_arch') - self.assertEqual(wt.reason, "{ os_arch : x64, } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'os_architecture') + self.assertEqual( + exception.reason, + "{ os_architecture : x64, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_arch_should_not_be_empty(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_architecture = '' + self.mox.ReplayAll() + + with self.assertRaises(NullFieldException) as nf: + nova_verifier._verify_validity(exist, 'all') + exception = nf.exception + self.assertEqual(exception.field_name, 'os_architecture') + self.assertEqual( + exception.reason, + "os_architecture field was null for exist id 23") self.mox.VerifyAll() def test_should_verify_os_distro_should_be_alphanumeric(self): @@ -483,16 +525,38 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.deleted_at = decimal.Decimal('5.1') exist.instance_type_id = 4 exist.rax_options = 12 - exist.os_arch = 'x64' + exist.os_architecture = 'x64' exist.os_distro = 'com.microsoft.server,' self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'os_distro') - self.assertEqual(wt.reason, "{ os_distro : com.microsoft.server, } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'os_distro') + self.assertEqual( + exception.reason, + "{ os_distro : com.microsoft.server, } of incorrect type for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_os_distro_should_not_be_empty(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = 4 + exist.rax_options = 12 + exist.os_architecture = 'x64' + exist.os_distro = '' + self.mox.ReplayAll() + + with self.assertRaises(NullFieldException) as nf: + nova_verifier._verify_validity(exist, 'all') + exception = nf.exception + self.assertEqual(exception.field_name, 'os_distro') + self.assertEqual( + exception.reason, + "os_distro field was null for exist id 23") self.mox.VerifyAll() def test_should_verify_os_version_should_be_alphanumeric(self): @@ -503,20 +567,21 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.deleted_at = decimal.Decimal('5.1') exist.instance_type_id = 4 exist.rax_options = 12 - exist.os_arch = 'x64' + exist.os_architecture = 'x64' exist.os_distro = 'com.microsoft.server' exist.os_version = '2008.2,' self.mox.ReplayAll() - try: - nova_verifier._verify_validity(exist) - self.fail() - except WrongTypeException as wt: - self.assertEqual(wt.field_name, 'os_version') - self.assertEqual(wt.reason, "{ os_version : 2008.2, } of incorrect type for exist id 23") + with self.assertRaises(WrongTypeException) as wt: + nova_verifier._verify_validity(exist, 'all') + exception = wt.exception + self.assertEqual(exception.field_name, 'os_version') + self.assertEqual( + exception.reason, + "{ os_version : 2008.2, } of incorrect type for exist id 23") self.mox.VerifyAll() - def test_should_verify_exist_fields_correctly(self): + def test_should_verify_os_version_should_not_be_empty(self): exist = self.mox.CreateMockAnything() exist.tenant = '3762854cd6f6435998188d5120e4c271' exist.id = 23 @@ -524,15 +589,56 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.deleted_at = decimal.Decimal('5.1') exist.instance_type_id = 4 exist.rax_options = 12 - exist.os_arch = 'x64' + exist.os_architecture = 'x64' + exist.os_distro = 'com.microsoft.server' + exist.os_version = '' + self.mox.ReplayAll() + + with self.assertRaises(NullFieldException) as nf: + nova_verifier._verify_validity(exist, 'all') + exception = nf.exception + self.assertEqual(exception.field_name, 'os_version') + self.assertEqual( + exception.reason, "os_version field was null for exist id 23") + self.mox.VerifyAll() + + def test_should_verify_all_exist_fields_when_validity_check_value_is_all(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = '4' + exist.rax_options = '12' + exist.os_architecture = 'x64' exist.os_distro = 'com.microsoft.server' exist.os_version = '2008.2' self.mox.ReplayAll() - nova_verifier._verify_validity(exist) + nova_verifier._verify_validity(exist, 'all') self.mox.VerifyAll() - def test_should_verify_exist_fields_correctly_and_deleted_at_can_be_none(self): + def test_should_verify_only_basic_exist_fields_when_validity_check_value_is_basic(self): + exist = self.mox.CreateMockAnything() + exist.tenant = '3762854cd6f6435998188d5120e4c271' + exist.id = 23 + exist.launched_at = decimal.Decimal('1.1') + exist.deleted_at = decimal.Decimal('5.1') + exist.instance_type_id = '4' + self.mox.ReplayAll() + + nova_verifier._verify_validity(exist, 'basic') + self.mox.VerifyAll() + + def test_should_not_verify_any_fields_if_validity_check_value_is_none(self): + exist = self.mox.CreateMockAnything() + exist.id = 23 + self.mox.ReplayAll() + + nova_verifier._verify_validity(exist, 'none') + self.mox.VerifyAll() + + def test_should_verify_exist_fields_even_if_deleted_at_is_none(self): exist = self.mox.CreateMockAnything() exist.tenant = '3762854cd6f6435998188d5120e4c271' exist.id = 23 @@ -540,12 +646,12 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.deleted_at = None exist.instance_type_id = 4 exist.rax_options = 12 - exist.os_arch = 'x64' + exist.os_architecture = 'x64' exist.os_distro = 'com.microsoft.server' exist.os_version = '2008.2' self.mox.ReplayAll() - nova_verifier._verify_validity(exist) + nova_verifier._verify_validity(exist, 'all') self.mox.VerifyAll() def test_verify_for_delete(self): @@ -618,12 +724,11 @@ class NovaVerifierTestCase(StacktachBaseTestCase): self.mox.ReplayAll() - try: + with self.assertRaises(VerificationException) as ve: nova_verifier._verify_for_delete(exist) - self.fail() - except VerificationException, ve: - msg = 'Found InstanceDeletes for non-delete exist' - self.assertEqual(ve.reason, msg) + exception = ve.exception + msg = 'Found InstanceDeletes for non-delete exist' + self.assertEqual(exception.reason, msg) self.mox.VerifyAll() @@ -636,13 +741,12 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.delete.deleted_at = decimal.Decimal('5.1') self.mox.ReplayAll() - try: + with self.assertRaises(FieldMismatch) as fm: nova_verifier._verify_for_delete(exist) - self.fail() - except FieldMismatch, fm: - self.assertEqual(fm.field_name, 'launched_at') - self.assertEqual(fm.expected, decimal.Decimal('1.1')) - self.assertEqual(fm.actual, decimal.Decimal('2.1')) + exception = fm.exception + self.assertEqual(exception.field_name, 'launched_at') + self.assertEqual(exception.expected, decimal.Decimal('1.1')) + self.assertEqual(exception.actual, decimal.Decimal('2.1')) self.mox.VerifyAll() def test_verify_for_delete_deleted_at_mismatch(self): @@ -654,13 +758,12 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist.delete.deleted_at = decimal.Decimal('6.1') self.mox.ReplayAll() - try: + with self.assertRaises(FieldMismatch) as fm: nova_verifier._verify_for_delete(exist) - self.fail() - except FieldMismatch, fm: - self.assertEqual(fm.field_name, 'deleted_at') - self.assertEqual(fm.expected, decimal.Decimal('5.1')) - self.assertEqual(fm.actual, decimal.Decimal('6.1')) + exception = fm.exception + self.assertEqual(exception.field_name, 'deleted_at') + self.assertEqual(exception.expected, decimal.Decimal('5.1')) + self.assertEqual(exception.actual, decimal.Decimal('6.1')) self.mox.VerifyAll() def test_verify_with_reconciled_data(self): @@ -799,10 +902,10 @@ class NovaVerifierTestCase(StacktachBaseTestCase): self.mox.StubOutWithMock(exist, 'mark_verified') nova_verifier._verify_for_launch(exist) nova_verifier._verify_for_delete(exist) - nova_verifier._verify_validity(exist) + nova_verifier._verify_validity(exist, 'all') exist.mark_verified() self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'all') self.assertTrue(result) self.mox.VerifyAll() @@ -817,7 +920,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): nova_verifier._verify_with_reconciled_data(exist)\ .AndRaise(NotFound('InstanceReconcile', {})) self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'all') self.assertFalse(result) self.mox.VerifyAll() @@ -833,7 +936,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): .AndRaise(VerificationException('test2')) exist.mark_failed(reason='test2') self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'none') self.assertFalse(result) self.mox.VerifyAll() @@ -850,7 +953,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): .AndRaise(NotFound('InstanceReconcile', {})) exist.mark_failed(reason='test') self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'none') self.assertFalse(result) self.mox.VerifyAll() @@ -866,7 +969,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): nova_verifier._verify_with_reconciled_data(exist) exist.mark_verified(reconciled=True) self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'none') self.assertTrue(result) self.mox.VerifyAll() @@ -883,7 +986,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): .AndRaise(Exception()) exist.mark_failed(reason='Exception') self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'none') self.assertFalse(result) self.mox.VerifyAll() @@ -901,7 +1004,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): .AndRaise(NotFound('InstanceReconcile', {})) exist.mark_failed(reason='test') self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'none') self.assertFalse(result) self.mox.VerifyAll() @@ -914,7 +1017,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): nova_verifier._verify_for_launch(exist).AndRaise(Exception()) exist.mark_failed(reason='Exception') self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'none') self.assertFalse(result) self.mox.VerifyAll() @@ -928,7 +1031,7 @@ class NovaVerifierTestCase(StacktachBaseTestCase): nova_verifier._verify_for_delete(exist).AndRaise(Exception()) exist.mark_failed(reason='Exception') self.mox.ReplayAll() - result, exists = nova_verifier._verify(exist) + result, exists = nova_verifier._verify(exist, 'none') self.assertFalse(result) self.mox.VerifyAll() @@ -948,9 +1051,9 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist2.update_status('verifying') exist1.save() exist2.save() - self.pool.apply_async(nova_verifier._verify, args=(exist1,), + self.pool.apply_async(nova_verifier._verify, args=(exist1, 'all'), callback=None) - self.pool.apply_async(nova_verifier._verify, args=(exist2,), + self.pool.apply_async(nova_verifier._verify, args=(exist2, 'all'), callback=None) self.mox.ReplayAll() self.verifier.verify_for_range(when_max) @@ -973,9 +1076,9 @@ class NovaVerifierTestCase(StacktachBaseTestCase): exist2.update_status('verifying') exist1.save() exist2.save() - self.pool.apply_async(nova_verifier._verify, args=(exist1,), + self.pool.apply_async(nova_verifier._verify, args=(exist1, 'all'), callback=callback) - self.pool.apply_async(nova_verifier._verify, args=(exist2,), + self.pool.apply_async(nova_verifier._verify, args=(exist2, 'all'), callback=callback) self.mox.ReplayAll() self.verifier.verify_for_range(when_max, callback=callback) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 04b72cc..a940b96 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -163,6 +163,7 @@ class FakeVerifierConfig(object): self.durable_queue = lambda: durable_queue self.topics = lambda: topics self.enable_notifications = lambda: notifs + self.validation_level = lambda: 'all' def make_verifier_config(notifs): diff --git a/verifier/base_verifier.py b/verifier/base_verifier.py index 01f9209..b6bc46f 100644 --- a/verifier/base_verifier.py +++ b/verifier/base_verifier.py @@ -80,8 +80,15 @@ def _is_like_date(attr_name, attr_value, exist_id): raise WrongTypeException(attr_name, attr_value, exist_id) -def _is_int(attr_name, attr_value, exist_id): - if not isinstance(attr_value, int): +def _is_long(attr_name, attr_value, exist_id): + if not isinstance(attr_value, long): + raise WrongTypeException(attr_name, attr_value, exist_id) + + +def _is_int_in_char(attr_name, attr_value, exist_id): + try: + int(attr_value) + except ValueError: raise WrongTypeException(attr_name, attr_value, exist_id) @@ -89,6 +96,7 @@ def _is_hex_owner_id(attr_name, attr_value, exist_id): if not re.match("[0-9a-f]{32}$", attr_value): raise WrongTypeException(attr_name, attr_value, exist_id) + def _is_alphanumeric(attr_name, attr_value, exist_id): if not re.match("[a-zA-Z0-9.]+$", attr_value): raise WrongTypeException(attr_name, attr_value, exist_id) diff --git a/verifier/config.py b/verifier/config.py index d48436f..f89b1f8 100644 --- a/verifier/config.py +++ b/verifier/config.py @@ -87,3 +87,7 @@ def password(): def virtual_host(): return config['rabbit']['virtual_host'] + + +def validation_level(): + return config['validation_level'] diff --git a/verifier/glance_verifier.py b/verifier/glance_verifier.py index 0ba611c..3ed49b6 100644 --- a/verifier/glance_verifier.py +++ b/verifier/glance_verifier.py @@ -65,11 +65,10 @@ def _verify_validity(exist): raise NullFieldException(field_name, exist.id) base_verifier._is_like_uuid('uuid', exist.uuid, exist.id) base_verifier._is_like_date('created_at', exist.created_at, exist.id) - base_verifier._is_int('size', exist.size, exist.id) + base_verifier._is_long('size', exist.size, exist.id) base_verifier._is_hex_owner_id('owner', exist.owner, exist.id) - def _verify_for_usage(exist, usage=None): usage_type = "ImageUsage" if not usage and exist.usage: diff --git a/verifier/nova_verifier.py b/verifier/nova_verifier.py index 5a65e2f..f0e4d69 100644 --- a/verifier/nova_verifier.py +++ b/verifier/nova_verifier.py @@ -18,7 +18,6 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. -import argparse import datetime import json import os @@ -146,22 +145,50 @@ def _verify_for_delete(exist, delete=None, 'deleted_at', exist.deleted_at, delete.deleted_at) -def _verify_validity(exist): - fields = {exist.tenant: 'tenant', exist.launched_at: 'launched_at', +def _verify_basic_validity(exist): + fields = {exist.tenant: 'tenant', + exist.launched_at: 'launched_at', exist.instance_type_id: 'instance_type_id'} for (field_value, field_name) in fields.items(): if field_value is None: raise NullFieldException(field_name, exist.id) base_verifier._is_hex_owner_id('tenant', exist.tenant, exist.id) - base_verifier._is_int('instance_type_id', exist.instance_type_id, exist.id) + base_verifier._is_int_in_char('instance_type_id', exist.instance_type_id, + exist.id) base_verifier._is_like_date('launched_at', exist.launched_at, exist.id) if exist.deleted_at is not None: base_verifier._is_like_date('deleted_at', exist.deleted_at, exist.id) - base_verifier._is_int('rax_options', exist.rax_options, exist.id) - base_verifier._is_alphanumeric('os_arch', exist.os_arch, exist.id) + + +def _verify_optional_validity(exist): + fields = {exist.rax_options: 'rax_options', + exist.os_architecture: 'os_architecture', + exist.os_version: 'os_version', + exist.os_distro: 'os_distro'} + for (field_value, field_name) in fields.items(): + if field_value == '': + raise NullFieldException(field_name, exist.id) + base_verifier._is_int_in_char('rax_options', exist.rax_options, exist.id) + base_verifier._is_alphanumeric('os_architecture', exist.os_architecture, exist.id) base_verifier._is_alphanumeric('os_distro', exist.os_distro, exist.id) base_verifier._is_alphanumeric('os_version', exist.os_version, exist.id) +def verify_fields_not_null(exist_id, null_value, fields): + + for (field_value, field_name) in fields.items(): + print "value: %s, name = %s" % (field_value, field_name) + if field_value == null_value: + raise NullFieldException(field_name, exist_id) + + +def _verify_validity(exist, validation_level): + if validation_level == 'none': + return + if validation_level == 'basic': + _verify_basic_validity(exist) + if validation_level == 'all': + _verify_basic_validity(exist) + _verify_optional_validity(exist) def _verify_with_reconciled_data(exist): @@ -212,15 +239,15 @@ def _attempt_reconciled_verify(exist, orig_e): return verified -def _verify(exist): +def _verify(exist, validation_level): verified = False try: if not exist.launched_at: raise VerificationException("Exists without a launched_at") + _verify_validity(exist, validation_level) _verify_for_launch(exist) _verify_for_delete(exist) - _verify_validity(exist) verified = True exist.mark_verified() @@ -267,8 +294,9 @@ class NovaVerifier(base_verifier.Verifier): for exist in exists[0:1000]: exist.update_status(models.InstanceExists.VERIFYING) exist.save() + validation_level = self.config.validation_level() result = self.pool.apply_async( - _verify, args=(exist,), + _verify, args=(exist, validation_level), callback=callback) self.results.append(result) added += 1 From 44a0826b877e55004430a26b6f8f46bb2ccf18ec Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Thu, 12 Sep 2013 15:06:09 -0400 Subject: [PATCH 03/16] Returning stacky search results in descending order --- stacktach/stacky_server.py | 2 +- tests/unit/test_stacky_server.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/stacktach/stacky_server.py b/stacktach/stacky_server.py index 296b9fb..9dc8e02 100644 --- a/stacktach/stacky_server.py +++ b/stacktach/stacky_server.py @@ -594,7 +594,7 @@ def search(request): results = [] try: - events = model_search(request, model, filters) + events = model_search(request, model, filters, order_by='-when') for event in events: when = dt.dt_from_decimal(event.when) routing_key_status = routing_key_type(event.routing_key) diff --git a/tests/unit/test_stacky_server.py b/tests/unit/test_stacky_server.py index c76ce31..c8f902d 100644 --- a/tests/unit/test_stacky_server.py +++ b/tests/unit/test_stacky_server.py @@ -1279,7 +1279,9 @@ class StackyServerTestCase(StacktachBaseTestCase): fake_request = self.mox.CreateMockAnything() fake_request.GET = {'field': 'tenant', 'value': 'tenant'} raw = self._create_raw() - models.RawData.objects.filter(tenant='tenant').AndReturn([raw]) + results = self.mox.CreateMockAnything() + models.RawData.objects.filter(tenant='tenant').AndReturn(results) + results.order_by('-when').AndReturn([raw]) raw.search_results([], mox.IgnoreArg(), ' ').AndReturn(search_result) self.mox.ReplayAll() @@ -1300,9 +1302,11 @@ class StackyServerTestCase(StacktachBaseTestCase): 'when_min': '1.1', 'when_max': '2.1'} raw = self._create_raw() + results = self.mox.CreateMockAnything() models.RawData.objects.filter(tenant='tenant', when__gte=decimal.Decimal('1.1'), - when__lte=decimal.Decimal('2.1')).AndReturn([raw]) + when__lte=decimal.Decimal('2.1')).AndReturn(results) + results.order_by('-when').AndReturn([raw]) raw.search_results([], mox.IgnoreArg(), ' ').AndReturn(search_result) self.mox.ReplayAll() @@ -1332,8 +1336,9 @@ class StackyServerTestCase(StacktachBaseTestCase): raw3 = self._create_raw() raw2.id = 2 raw3.id = 3 - models.RawData.objects.filter(tenant='tenant').AndReturn([raw1, raw2, - raw3]) + results = self.mox.CreateMockAnything() + models.RawData.objects.filter(tenant='tenant').AndReturn(results) + results.order_by('-when').AndReturn([raw1, raw2, raw3]) raw1.search_results([], mox.IgnoreArg(), ' ').AndReturn(search_result) raw2.search_results(search_result, mox.IgnoreArg(),' ').AndReturn(search_result_2) self.mox.ReplayAll() From d10409b81ea5043786bc635cac21d7a2d583db9c Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 10:53:05 -0400 Subject: [PATCH 04/16] Optimizing requests for uuid query in error detail --- reports/error_details.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/reports/error_details.py b/reports/error_details.py index a45bfd5..c3b7b8e 100644 --- a/reports/error_details.py +++ b/reports/error_details.py @@ -160,16 +160,24 @@ if __name__ == '__main__': expiry = 60 * 60 # 1 hour cmds = ['create', 'rebuild', 'rescue', 'resize', 'snapshot'] + requests = models.RawData.objects.filter(when__gt=dstart, when__lte=dend)\ + .exclude(instance=None)\ + .values('request_id', 'instance')\ + .distinct() + inst_recs = {} + for request in requests: + uuid = request['instance'] + request_id = request['request_id'] + if uuid in inst_recs: + inst_recs[uuid].append(request_id) + else: + inst_recs[uuid] = [request_id] + for uuid_dict in updates: uuid = uuid_dict['instance'] - # All the unique Request ID's for this instance during that timespan. - reqs = models.RawData.objects.filter(instance=uuid, - when__gt=dstart, when__lte=dend)\ - .values('request_id').distinct() - req_list = [] - for req_dict in reqs: + for req_dict in inst_recs.get(uuid, []): req = req_dict['request_id'] raws = list(models.RawData.objects.filter(request_id=req) From 3441f58ba2936fef28b2d8c6a892f3b9bc4a9be3 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 11:00:14 -0400 Subject: [PATCH 05/16] Excluding exists from error_details instance requests query --- reports/error_details.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reports/error_details.py b/reports/error_details.py index c3b7b8e..0842471 100644 --- a/reports/error_details.py +++ b/reports/error_details.py @@ -161,7 +161,8 @@ if __name__ == '__main__': cmds = ['create', 'rebuild', 'rescue', 'resize', 'snapshot'] requests = models.RawData.objects.filter(when__gt=dstart, when__lte=dend)\ - .exclude(instance=None)\ + .exclude(instance=None, + event='compute.instance.exists')\ .values('request_id', 'instance')\ .distinct() inst_recs = {} From a23864cc9aaf54aedab274d7292f72266070b104 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 11:52:18 -0400 Subject: [PATCH 06/16] Adding travis config --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c52c8f5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "2.6" + - "2.7" +# command to install dependencies +# install: "pip install -r requirements.txt --use-mirrors" +# # command to run tests +script: "./run_tests_venv.sh" From 1c8afed7aade80441afd7f44b95478b64589747a Mon Sep 17 00:00:00 2001 From: Thomas Maddox Date: Fri, 13 Sep 2013 11:08:34 -0500 Subject: [PATCH 07/16] fixed request_id and added to pretty.py as well --- reports/error_details.py | 4 +--- reports/pretty.py | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/reports/error_details.py b/reports/error_details.py index 0842471..590f4b8 100644 --- a/reports/error_details.py +++ b/reports/error_details.py @@ -178,9 +178,7 @@ if __name__ == '__main__': uuid = uuid_dict['instance'] req_list = [] - for req_dict in inst_recs.get(uuid, []): - req = req_dict['request_id'] - + for req in inst_recs.get(uuid, []): raws = list(models.RawData.objects.filter(request_id=req) .exclude(event='compute.instance.exists') .values("id", "when", "routing_key", "old_state", diff --git a/reports/pretty.py b/reports/pretty.py index 654fc40..1303f5f 100644 --- a/reports/pretty.py +++ b/reports/pretty.py @@ -52,6 +52,21 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=97, expiry = 60 * 60 # 1 hour cmds = ['create', 'rebuild', 'rescue', 'resize', 'snapshot'] + requests = models.RawData.objects.filter(when__gt=dstart, when__lte=dend)\ + .exclude(instance=None, + event='compute.instance.exists')\ + .values('request_id', 'instance')\ + .distinct() + inst_recs = {} + for request in requests: + uuid = request['instance'] + request_id = request['request_id'] + if uuid in inst_recs: + inst_recs[uuid].append(request_id) + else: + inst_recs[uuid] = [request_id] + + failures = {} # { key : {failure_type: count} } durations = {} attempts = {} @@ -59,14 +74,7 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=97, for uuid_dict in updates: uuid = uuid_dict['instance'] - # All the unique Request ID's for this instance during that timespan. - reqs = models.RawData.objects.filter(instance=uuid, - when__gt=dstart, when__lte=dend) \ - .values('request_id').distinct() - - - for req_dict in reqs: - req = req_dict['request_id'] + for req in inst_recs.get(uuid, []): raws = models.RawData.objects.filter(request_id=req)\ .exclude(event='compute.instance.exists')\ .order_by('when') From dfb591683315c6b999b16232607fba9af5f6ea25 Mon Sep 17 00:00:00 2001 From: Thomas Maddox Date: Fri, 13 Sep 2013 11:25:32 -0500 Subject: [PATCH 08/16] small tweak --- reports/error_details.py | 7 +++---- reports/pretty.py | 8 +++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/reports/error_details.py b/reports/error_details.py index 590f4b8..9cc83d1 100644 --- a/reports/error_details.py +++ b/reports/error_details.py @@ -169,10 +169,9 @@ if __name__ == '__main__': for request in requests: uuid = request['instance'] request_id = request['request_id'] - if uuid in inst_recs: - inst_recs[uuid].append(request_id) - else: - inst_recs[uuid] = [request_id] + value = inst_recs.get(uuid, []) + value.append(request_id) + inst_recs[uuid] = value for uuid_dict in updates: uuid = uuid_dict['instance'] diff --git a/reports/pretty.py b/reports/pretty.py index 1303f5f..1cd1fe8 100644 --- a/reports/pretty.py +++ b/reports/pretty.py @@ -61,11 +61,9 @@ def make_report(yesterday=None, start_hour=0, hours=24, percentile=97, for request in requests: uuid = request['instance'] request_id = request['request_id'] - if uuid in inst_recs: - inst_recs[uuid].append(request_id) - else: - inst_recs[uuid] = [request_id] - + value = inst_recs.get(uuid, []) + value.append(request_id) + inst_recs[uuid] = value failures = {} # { key : {failure_type: count} } durations = {} From b6cd2ba20ca35934da2ffa9fec797cc8dcc2708a Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 12:59:01 -0400 Subject: [PATCH 09/16] Delete root __init__ before travis runs tests --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c52c8f5..93ab512 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,7 @@ python: # command to install dependencies # install: "pip install -r requirements.txt --use-mirrors" # # command to run tests +before_script: + - rm -f __init__.py + - rm -f __init__.pyc script: "./run_tests_venv.sh" From 7660bbd222e59964d6bcf98e591f5abbe1bdb159 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 13:09:29 -0400 Subject: [PATCH 10/16] Fix for travis-ci/multiprocessing --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93ab512..9de9299 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,9 @@ python: # command to install dependencies # install: "pip install -r requirements.txt --use-mirrors" # # command to run tests +install: + - "sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm" before_script: - - rm -f __init__.py - - rm -f __init__.pyc + - "rm -f __init__.py" + - "rm -f __init__.pyc" script: "./run_tests_venv.sh" From 0bba81e9c6abd0e60ca63f716871e0d4b6dac0ba Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 13:18:02 -0400 Subject: [PATCH 11/16] Adding build status to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 493c775..6343c54 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.org/rackerlabs/stacktach.png?branch=master)](https://travis-ci.org/rackerlabs/stacktach) + # StackTach StackTach is a debugging / monitoring utility for OpenStack ([Open]StackTach[ometer]). StackTach can work with multiple datacenters including multi-cell deployments. From 9ce61c8e552148f1c1be9f6a2ad19e4cfbcb3f77 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 13:21:33 -0400 Subject: [PATCH 12/16] Disabling travis emails --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9de9299..5559186 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,3 +11,6 @@ before_script: - "rm -f __init__.py" - "rm -f __init__.pyc" script: "./run_tests_venv.sh" + +notifications: + email: false From eadf56642f3e37e574abd2829b6da4b677fe3606 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 14:56:30 -0400 Subject: [PATCH 13/16] Handling case of no data in 0003 migration --- ...tanceexists_and_instanceusages_from_rawdata.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py index 179811c..3688951 100644 --- a/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py +++ b/stacktach/migrations/0003_populate_usage_related_fields_in_rawdataimagemeta_instanceexists_and_instanceusages_from_rawdata.py @@ -41,13 +41,14 @@ def queryset_iterator(queryset, chunksize=1000): Note that the implementation of the iterator does not support ordered query sets. ''' id = 0 - last_pk = queryset.order_by('-id')[0]['id'] - queryset = queryset.order_by('id') - while id < last_pk: - for row in queryset.filter(id__gt=id)[:chunksize]: - id = row['id'] - yield row - gc.collect() + if queryset.order_by('-id').count() > 0: + last_pk = queryset.order_by('-id')[0]['id'] + queryset = queryset.order_by('id') + while id < last_pk: + for row in queryset.filter(id__gt=id)[:chunksize]: + id = row['id'] + yield row + gc.collect() class Migration(DataMigration): From f81383e22a6ff2f936c321a03aeab0c7c3a57816 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Fri, 13 Sep 2013 16:31:51 -0400 Subject: [PATCH 14/16] Consuming rescue notifications from Nova for Usage --- stacktach/views.py | 11 +++- tests/unit/test_stacktach.py | 103 +++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/stacktach/views.py b/stacktach/views.py index a6699bf..7746946 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -158,6 +158,8 @@ INSTANCE_EVENT = { 'resize_revert_start': 'compute.instance.resize.revert.start', 'resize_revert_end': 'compute.instance.resize.revert.end', 'resize_finish_end': 'compute.instance.finish_resize.end', + 'rescue_start': 'compute.instance.rescue.start', + 'rescue_end': 'compute.instance.rescue.end', 'delete_end': 'compute.instance.delete.end', 'exists': 'compute.instance.exists', } @@ -171,12 +173,14 @@ def _process_usage_for_new_launch(raw, notification): (usage, new) = STACKDB.get_or_create_instance_usage(**values) if raw.event in [INSTANCE_EVENT['create_start'], - INSTANCE_EVENT['rebuild_start']]: + INSTANCE_EVENT['rebuild_start'], + INSTANCE_EVENT['rescue_start']]: usage.instance_type_id = notification.instance_type_id if raw.event in [INSTANCE_EVENT['rebuild_start'], INSTANCE_EVENT['resize_prep_start'], - INSTANCE_EVENT['resize_revert_start']] and\ + INSTANCE_EVENT['resize_revert_start'], + INSTANCE_EVENT['rescue_start']] and\ usage.launched_at is None: # Grab the launched_at so if this action spans the audit period, # we will have a launch record corresponding to the exists. @@ -205,7 +209,8 @@ def _process_usage_for_updates(raw, notification): if raw.event in [INSTANCE_EVENT['create_end'], INSTANCE_EVENT['rebuild_end'], INSTANCE_EVENT['resize_finish_end'], - INSTANCE_EVENT['resize_revert_end']]: + INSTANCE_EVENT['resize_revert_end'], + INSTANCE_EVENT['rescue_end']]: usage.launched_at = utils.str_time_to_unix(notification.launched_at) if raw.event == INSTANCE_EVENT['resize_revert_end']: diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index b770cd9..e1a9ebf 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -336,6 +336,39 @@ class StacktachUsageParsingTestCase(StacktachBaseTestCase): self.mox.VerifyAll() + def test_process_usage_for_new_launch_rescue_start(self): + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.rescue.start' + + usage = self.mox.CreateMockAnything() + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) + + self.assertEquals(usage.instance_type_id, INSTANCE_TYPE_ID_1) + self.assertEquals(usage.tenant, TENANT_ID_1) + self.assertEquals(usage.os_architecture, OS_ARCH_1) + self.assertEquals(usage.os_version, OS_VERSION_1) + self.assertEquals(usage.os_distro, OS_DISTRO_1) + self.assertEquals(usage.rax_options, RAX_OPTIONS_1) + + self.mox.VerifyAll() + def test_process_usage_for_new_launch_rebuild_start(self): notification = self.mox.CreateMockAnything() notification.launched_at = str(DUMMY_TIME) @@ -506,6 +539,41 @@ class StacktachUsageParsingTestCase(StacktachBaseTestCase): self.mox.VerifyAll() + def test_process_usage_for_new_launch_rescue_start_when_launched_at_in_db(self): + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.rescue.start' + + orig_launched_at = utils.decimal_utc(DUMMY_TIME - datetime.timedelta(days=1)) + usage = self.mox.CreateMockAnything() + usage.launched_at = orig_launched_at + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_new_launch(raw, notification) + + self.assertEqual(usage.launched_at, orig_launched_at) + self.assertEqual(usage.tenant, TENANT_ID_1) + self.assertEquals(usage.os_architecture, OS_ARCH_1) + self.assertEquals(usage.os_version, OS_VERSION_1) + self.assertEquals(usage.os_distro, OS_DISTRO_1) + self.assertEquals(usage.rax_options, RAX_OPTIONS_1) + + self.mox.VerifyAll() + def test_process_usage_for_updates_create_end(self): notification = self.mox.CreateMockAnything() notification.launched_at = str(DUMMY_TIME) @@ -541,6 +609,41 @@ class StacktachUsageParsingTestCase(StacktachBaseTestCase): self.mox.VerifyAll() + def test_process_usage_for_updates_rescue_end(self): + notification = self.mox.CreateMockAnything() + notification.launched_at = str(DUMMY_TIME) + notification.tenant = TENANT_ID_1 + notification.rax_options = RAX_OPTIONS_1 + notification.os_architecture = OS_ARCH_1 + notification.os_version = OS_VERSION_1 + notification.os_distro = OS_DISTRO_1 + notification.instance = INSTANCE_ID_1 + notification.request_id = REQUEST_ID_1 + notification.instance_type_id = INSTANCE_TYPE_ID_1 + notification.message = None + + raw = self.mox.CreateMockAnything() + raw.event = 'compute.instance.rescue.end' + + usage = self.mox.CreateMockAnything() + usage.launched_at = None + views.STACKDB.get_or_create_instance_usage(instance=INSTANCE_ID_1, + request_id=REQUEST_ID_1) \ + .AndReturn((usage, True)) + views.STACKDB.save(usage) + self.mox.ReplayAll() + + views._process_usage_for_updates(raw, notification) + + self.assertEqual(usage.launched_at, utils.decimal_utc(DUMMY_TIME)) + self.assertEqual(usage.tenant, TENANT_ID_1) + self.assertEquals(usage.os_architecture, OS_ARCH_1) + self.assertEquals(usage.os_version, OS_VERSION_1) + self.assertEquals(usage.os_distro, OS_DISTRO_1) + self.assertEquals(usage.rax_options, RAX_OPTIONS_1) + + self.mox.VerifyAll() + def test_process_usage_for_updates_create_end_success_message(self): notification = self.mox.CreateMockAnything() notification.launched_at = str(DUMMY_TIME) From 0d1ac8682a6dbaad52bb88cd86df839bc5f85845 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 16 Sep 2013 13:41:04 -0400 Subject: [PATCH 15/16] Mapping rescue events to process functions --- stacktach/views.py | 2 ++ tests/unit/test_stacktach.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/stacktach/views.py b/stacktach/views.py index 7746946..1342e46 100644 --- a/stacktach/views.py +++ b/stacktach/views.py @@ -306,11 +306,13 @@ USAGE_PROCESS_MAPPING = { INSTANCE_EVENT['rebuild_start']: _process_usage_for_new_launch, INSTANCE_EVENT['resize_prep_start']: _process_usage_for_new_launch, INSTANCE_EVENT['resize_revert_start']: _process_usage_for_new_launch, + INSTANCE_EVENT['rescue_start']: _process_usage_for_new_launch, INSTANCE_EVENT['create_end']: _process_usage_for_updates, INSTANCE_EVENT['rebuild_end']: _process_usage_for_updates, INSTANCE_EVENT['resize_prep_end']: _process_usage_for_updates, INSTANCE_EVENT['resize_finish_end']: _process_usage_for_updates, INSTANCE_EVENT['resize_revert_end']: _process_usage_for_updates, + INSTANCE_EVENT['rescue_end']: _process_usage_for_updates, INSTANCE_EVENT['delete_end']: _process_delete, INSTANCE_EVENT['exists']: _process_exists } diff --git a/tests/unit/test_stacktach.py b/tests/unit/test_stacktach.py index e1a9ebf..3c8da30 100644 --- a/tests/unit/test_stacktach.py +++ b/tests/unit/test_stacktach.py @@ -303,6 +303,11 @@ class StacktachUsageParsingTestCase(StacktachBaseTestCase): else: stacklog.get_logger(name=name).AndReturn(self.log) + def test_all_instance_events_have_mapping(self): + for key, value in views.INSTANCE_EVENT.items(): + msg = "'%s' does not have a process function mapping." % value + self.assertTrue(value in views.USAGE_PROCESS_MAPPING, msg) + def test_process_usage_for_new_launch_create_start(self): notification = self.mox.CreateMockAnything() notification.launched_at = str(DUMMY_TIME) From 9f58358860ce8a784a8c4a164e380dbb1e471a1c Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 16 Sep 2013 15:45:44 -0400 Subject: [PATCH 16/16] End verifier if there are no children --- verifier/start_verifier.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/verifier/start_verifier.py b/verifier/start_verifier.py index f35c851..7f3e8c1 100644 --- a/verifier/start_verifier.py +++ b/verifier/start_verifier.py @@ -84,6 +84,10 @@ if __name__ == '__main__': process = Process(target=make_and_start_verifier, args=(exchange,)) process.start() processes.append(process) - signal.signal(signal.SIGINT, kill_time) - signal.signal(signal.SIGTERM, kill_time) - signal.pause() + + if len(processes) > 0: + # Only pause parent process if there are children running. + # Otherwise just end... + signal.signal(signal.SIGINT, kill_time) + signal.signal(signal.SIGTERM, kill_time) + signal.pause()