diff --git a/stacktach/models.py b/stacktach/models.py index f3db5a3..f76fed9 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -108,6 +108,7 @@ class InstanceUsage(models.Model): raw = raws[0] return raw.deployment + class InstanceDeletes(models.Model): instance = models.CharField(max_length=50, null=True, blank=True, db_index=True) @@ -134,6 +135,12 @@ class InstanceReconcile(models.Model): null=True, blank=True, db_index=True) + tenant = models.CharField(max_length=50, null=True, blank=True, + db_index=True) + os_architecture = models.TextField(null=True, blank=True) + os_distro = models.TextField(null=True, blank=True) + os_version = models.TextField(null=True, blank=True) + rax_options = models.TextField(null=True, blank=True) source = models.CharField(max_length=150, null=True, blank=True, db_index=True) diff --git a/stacktach/reconciler/__init__.py b/stacktach/reconciler/__init__.py index 3e5dcaf..62806ef 100644 --- a/stacktach/reconciler/__init__.py +++ b/stacktach/reconciler/__init__.py @@ -23,7 +23,6 @@ import json from stacktach import models from stacktach.reconciler import exceptions from stacktach.reconciler import nova -from stacktach import datetime_to_decimal as dt DEFAULT_CLIENT = nova.JSONBridgeClient @@ -73,18 +72,48 @@ class Reconciler(object): else: return False - def _reconcile_instance(self, usage, src, - launched_at=None, deleted_at=None, - instance_type_id=None): + def _reconcile_instance(self, usage, src, deleted_at=None): values = { 'instance': usage.instance, - 'launched_at': (launched_at or usage.launched_at), + 'launched_at': usage.launched_at, 'deleted_at': deleted_at, - 'instance_type_id': (instance_type_id or usage.instance_type_id), + 'instance_type_id': usage.instance_type_id, 'source': 'reconciler:%s' % src, + 'tenant': usage.tenant, + 'os_architecture': usage.os_architecture, + 'os_distro': usage.os_distro, + 'os_version': usage.os_version, + 'rax_options': usage.rax_options, } models.InstanceReconcile(**values).save() + def _fields_match(self, exists, instance): + match = True + + if (exists.launched_at != instance['launched_at'] or + exists.instance_type_id != instance['instance_type_id'] or + exists.tenant != instance['tenant'] or + exists.os_architecture != instance['os_architecture'] or + exists.os_distro != instance['os_distro'] or + exists.os_version != instance['os_version'] or + exists.rax_options != instance['rax_options']): + match = False + + if exists.deleted_at is not None: + # Exists says deleted + if (instance['deleted'] and + exists.deleted_at != instance['deleted_at']): + # Nova says deleted, but times don't match + match = False + elif not instance['deleted']: + # Nova says not deleted + match = False + elif exists.deleted_at is None and instance['deleted']: + # Exists says not deleted, but Nova says not deleted + match = False + + return match + def missing_exists_for_instance(self, launched_id, period_beginning): reconciled = False @@ -108,24 +137,15 @@ class Reconciler(object): return reconciled def failed_validation(self, exists): - reconcilable = False + reconciled = False region = self._region_for_usage(exists) - deleted_at = None try: instance = self.client.get_instance(region, exists.instance) - if (instance['launched_at'] == exists.launched_at and - instance['instance_type_id'] == exists.instance_type_id): - if instance['deleted'] and exists.deleted_at is not None: - if instance['deleted_at'] == exists.deleted_at: - deleted_at = exists.deleted_at - reconcilable = True - elif not instance['deleted'] and exists.deleted_at is None: - reconcilable = True + if self._fields_match(exists, instance): + self._reconcile_instance(exists, self.client.src_str, + deleted_at=exists.deleted_at) + reconciled = True except exceptions.NotFound: - reconcilable = False + pass - if reconcilable: - self._reconcile_instance(exists, self.client.src_str, - deleted_at=deleted_at) - - return reconcilable + return reconciled diff --git a/stacktach/reconciler/nova.py b/stacktach/reconciler/nova.py index 59e5955..5850fec 100644 --- a/stacktach/reconciler/nova.py +++ b/stacktach/reconciler/nova.py @@ -1,13 +1,35 @@ -import json +import os, sys import requests +sys.path.append('/home/andrewmelton/publicgit/stacktach_app') + from stacktach import utils as stackutils from stacktach.reconciler import exceptions from stacktach.reconciler.utils import empty_reconciler_instance - GET_INSTANCE_QUERY = "SELECT * FROM instances where uuid ='%s';" +GET_INSTANCE_SYSTEM_METADATA = """ +SELECT * FROM instance_system_metadata + WHERE instance_uuid = '%s' AND + deleted = 0 AND `key` IN ('image_org.openstack__1__architecture', + 'image_org.openstack__1__os_distro', + 'image_org.openstack__1__os_version', + 'image_com.rackspace__1__options'); +""" +METADATA_MAPPING = { + 'image_org.openstack__1__architecture': 'os_architecture', + 'image_org.openstack__1__os_distro': 'os_distro', + 'image_org.openstack__1__os_version': 'os_version', + 'image_com.rackspace__1__options': 'rax_options', +} + + +def _json(result): + if callable(result.json): + return result.json() + else: + return result.json class JSONBridgeClient(object): @@ -22,14 +44,15 @@ class JSONBridgeClient(object): def _do_query(self, region, query): data = {'sql': query} credentials = (self.config['username'], self.config['password']) - return requests.post(self._url_for_region(region), data, - verify=False, auth=credentials).json() + return _json(requests.post(self._url_for_region(region), data, + verify=False, auth=credentials)) - def _to_reconciler_instance(self, instance): + def _to_reconciler_instance(self, instance, metadata=None): r_instance = empty_reconciler_instance() r_instance.update({ 'id': instance['uuid'], - 'instance_type_id': instance['instance_type_id'], + 'tenant': instance['project_id'], + 'instance_type_id': str(instance['instance_type_id']), }) if instance['launched_at'] is not None: @@ -43,12 +66,41 @@ class JSONBridgeClient(object): if instance['deleted'] != 0: r_instance['deleted'] = True + if metadata is not None: + r_instance.update(metadata) + return r_instance - def get_instance(self, region, uuid): + def _get_instance_meta(self, region, uuid): + results = self._do_query(region, GET_INSTANCE_SYSTEM_METADATA % uuid) + metadata = {} + for result in results['result']: + key = result['key'] + if key in METADATA_MAPPING: + metadata[METADATA_MAPPING[key]] = result['value'] + return metadata + + def get_instance(self, region, uuid, get_metadata=False): results = self._do_query(region, GET_INSTANCE_QUERY % uuid)['result'] if len(results) > 0: - return self._to_reconciler_instance(results[0]) + metadata = None + if get_metadata: + metadata = self._get_instance_meta(region, uuid) + return self._to_reconciler_instance(results[0], metadata=metadata) else: msg = "Couldn't find instance (%s) using JSON Bridge in region (%s)" - raise exceptions.NotFound(msg % (uuid, region)) \ No newline at end of file + raise exceptions.NotFound(msg % (uuid, region)) + +if __name__ == '__main__': + json_bridge_config = { + 'url': 'http://devstack.ceilo-dev.ord.ohthree.com:8080/query/', + 'username': '', + 'password': '', + 'databases': { + 'RegionOne': 'nova', + } + } + client = JSONBridgeClient(json_bridge_config) + print client.get_instance('RegionOne', + 'e23ff37f-a02d-4c63-b11e-cc15fdced2cf', + get_metadata=True) diff --git a/stacktach/reconciler/utils.py b/stacktach/reconciler/utils.py index b835d25..26d0ac4 100644 --- a/stacktach/reconciler/utils.py +++ b/stacktach/reconciler/utils.py @@ -1,9 +1,14 @@ def empty_reconciler_instance(): r_instance = { 'id': None, + 'tenant': None, 'launched_at': None, 'deleted': False, 'deleted_at': None, - 'instance_type_ud': None + 'instance_type_ud': None, + 'os_architecture': None, + 'os_distro': None, + 'os_version': None, + 'rax_options': None, } return r_instance diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py index e854a26..c7d5725 100644 --- a/tests/unit/test_reconciler.py +++ b/tests/unit/test_reconciler.py @@ -29,14 +29,21 @@ from stacktach import reconciler from stacktach import utils as stackutils from stacktach.reconciler import exceptions from stacktach.reconciler import nova +from stacktach.reconciler import utils as rec_utils from tests.unit import utils from tests.unit.utils import INSTANCE_ID_1 +from tests.unit.utils import TENANT_ID_1 region_mapping = { 'RegionOne.prod.cell1': 'RegionOne', 'RegionTwo.prod.cell1': 'RegionTwo', } +DEFAULT_OS_ARCH = 'os_arch' +DEFAULT_OS_DISTRO = 'os_dist' +DEFAULT_OS_VERSION = "1.1" +DEFAULT_RAX_OPTIONS = "rax_ops" + class ReconcilerTestCase(unittest.TestCase): def setUp(self): @@ -75,16 +82,49 @@ class ReconcilerTestCase(unittest.TestCase): def tearDown(self): self.mox.UnsetStubs() + def _fake_usage(self, is_exists=False, is_deleted=False): + usage = self.mox.CreateMockAnything() + usage.id = 1 + beginning_d = utils.decimal_utc() + usage.instance = INSTANCE_ID_1 + launched_at = beginning_d - (60*60) + usage.launched_at = launched_at + usage.instance_type_id = 1 + usage.tenant = TENANT_ID_1 + if is_exists: + usage.deleted_at = None + if is_deleted: + usage.deleted_at = beginning_d + deployment = self.mox.CreateMockAnything() + deployment.name = 'RegionOne.prod.cell1' + usage.deployment().AndReturn(deployment) + usage.os_architecture = DEFAULT_OS_ARCH + usage.os_distro = DEFAULT_OS_DISTRO + usage.os_version = DEFAULT_OS_VERSION + usage.rax_options = DEFAULT_RAX_OPTIONS + return usage + def _fake_reconciler_instance(self, uuid=INSTANCE_ID_1, launched_at=None, deleted_at=None, deleted=False, - instance_type_id=1): - return { + instance_type_id=1, tenant=TENANT_ID_1, + os_arch=DEFAULT_OS_ARCH, + os_distro=DEFAULT_OS_DISTRO, + os_verison=DEFAULT_OS_VERSION, + rax_options=DEFAULT_RAX_OPTIONS): + instance = rec_utils.empty_reconciler_instance() + instance.update({ 'id': uuid, 'launched_at': launched_at, 'deleted_at': deleted_at, 'deleted': deleted, - 'instance_type_id': instance_type_id - } + 'instance_type_id': instance_type_id, + 'tenant': tenant, + 'os_architecture': os_arch, + 'os_distro': os_distro, + 'os_version': os_verison, + 'rax_options': rax_options, + }) + return instance def test_load_client_json_bridge(self): mock_config = self.mox.CreateMockAnything() @@ -139,17 +179,11 @@ class ReconcilerTestCase(unittest.TestCase): self.mox.VerifyAll() def test_missing_exists_for_instance(self): - launch_id = 1 - beginning_d = utils.decimal_utc() - launch = self.mox.CreateMockAnything() - launch.instance = INSTANCE_ID_1 - launch.launched_at = beginning_d - (60*60) - launch.instance_type_id = 1 - models.InstanceUsage.objects.get(id=launch_id).AndReturn(launch) - deployment = self.mox.CreateMockAnything() - launch.deployment().AndReturn(deployment) - deployment.name = 'RegionOne.prod.cell1' - deleted_at = beginning_d - (60*30) + launch = self._fake_usage() + launched_at = launch.launched_at + deleted_at = launched_at + (60*30) + period_beginning = deleted_at + 1 + models.InstanceUsage.objects.get(id=launch.id).AndReturn(launch) rec_inst = self._fake_reconciler_instance(deleted=True, deleted_at=deleted_at) self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst) @@ -158,14 +192,19 @@ class ReconcilerTestCase(unittest.TestCase): 'launched_at': launch.launched_at, 'deleted_at': deleted_at, 'instance_type_id': launch.instance_type_id, - 'source': 'reconciler:mocked_client' + 'source': 'reconciler:mocked_client', + 'tenant': TENANT_ID_1, + 'os_architecture': DEFAULT_OS_ARCH, + 'os_distro': DEFAULT_OS_DISTRO, + 'os_version': DEFAULT_OS_VERSION, + 'rax_options': DEFAULT_RAX_OPTIONS, } result = self.mox.CreateMockAnything() models.InstanceReconcile(**reconcile_vals).AndReturn(result) result.save() self.mox.ReplayAll() - result = self.reconciler.missing_exists_for_instance(launch_id, - beginning_d) + result = self.reconciler.missing_exists_for_instance(launch.id, + period_beginning) self.assertTrue(result) self.mox.VerifyAll() @@ -189,16 +228,8 @@ class ReconcilerTestCase(unittest.TestCase): self.mox.VerifyAll() def test_failed_validation(self): - beginning_d = utils.decimal_utc() - exists = self.mox.CreateMockAnything() - exists.instance = INSTANCE_ID_1 - launched_at = beginning_d - (60*60) - exists.launched_at = launched_at - exists.instance_type_id = 1 - exists.deleted_at = None - deployment = self.mox.CreateMockAnything() - exists.deployment().AndReturn(deployment) - deployment.name = 'RegionOne.prod.cell1' + exists = self._fake_usage(is_exists=True) + launched_at = exists.launched_at rec_inst = self._fake_reconciler_instance(launched_at=launched_at) self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst) reconcile_vals = { @@ -206,7 +237,12 @@ class ReconcilerTestCase(unittest.TestCase): 'launched_at': exists.launched_at, 'deleted_at': exists.deleted_at, 'instance_type_id': exists.instance_type_id, - 'source': 'reconciler:mocked_client' + 'source': 'reconciler:mocked_client', + 'tenant': TENANT_ID_1, + 'os_architecture': DEFAULT_OS_ARCH, + 'os_distro': DEFAULT_OS_DISTRO, + 'os_version': DEFAULT_OS_VERSION, + 'rax_options': DEFAULT_RAX_OPTIONS, } result = self.mox.CreateMockAnything() models.InstanceReconcile(**reconcile_vals).AndReturn(result) @@ -217,26 +253,24 @@ class ReconcilerTestCase(unittest.TestCase): self.mox.VerifyAll() def test_failed_validation_deleted(self): - beginning_d = utils.decimal_utc() - exists = self.mox.CreateMockAnything() - exists.instance = INSTANCE_ID_1 - launched_at = beginning_d - (60*60) - exists.launched_at = launched_at - exists.instance_type_id = 1 - exists.deleted_at = beginning_d - deployment = self.mox.CreateMockAnything() - exists.deployment().AndReturn(deployment) - deployment.name = 'RegionOne.prod.cell1' + exists = self._fake_usage(is_exists=True, is_deleted=True) + launched_at = exists.launched_at + deleted_at = exists.deleted_at rec_inst = self._fake_reconciler_instance(launched_at=launched_at, deleted=True, - deleted_at=beginning_d) + deleted_at=deleted_at) self.client.get_instance('RegionOne', INSTANCE_ID_1).AndReturn(rec_inst) reconcile_vals = { 'instance': exists.instance, 'launched_at': exists.launched_at, 'deleted_at': exists.deleted_at, 'instance_type_id': exists.instance_type_id, - 'source': 'reconciler:mocked_client' + 'source': 'reconciler:mocked_client', + 'tenant': TENANT_ID_1, + 'os_architecture': DEFAULT_OS_ARCH, + 'os_distro': DEFAULT_OS_DISTRO, + 'os_version': DEFAULT_OS_VERSION, + 'rax_options': DEFAULT_RAX_OPTIONS, } result = self.mox.CreateMockAnything() models.InstanceReconcile(**reconcile_vals).AndReturn(result) @@ -333,13 +367,15 @@ class NovaJSONBridgeClientTestCase(unittest.TestCase): response.json().AndReturn(result) def _fake_instance(self, uuid=INSTANCE_ID_1, launched_at=None, - terminated_at=None, deleted=0, instance_type_id=1): + terminated_at=None, deleted=0, instance_type_id=1, + project_id=TENANT_ID_1): return { 'uuid': uuid, 'launched_at': launched_at, 'terminated_at': terminated_at, 'deleted': deleted, - 'instance_type_id': instance_type_id + 'instance_type_id': instance_type_id, + 'project_id': project_id } def test_get_instance(self): @@ -355,7 +391,7 @@ class NovaJSONBridgeClientTestCase(unittest.TestCase): instance = self.client.get_instance('RegionOne', INSTANCE_ID_1) self.assertIsNotNone(instance) self.assertEqual(instance['id'], INSTANCE_ID_1) - self.assertEqual(instance['instance_type_id'], 1) + self.assertEqual(instance['instance_type_id'], '1') launched_at_dec = stackutils.str_time_to_unix(launched_at) self.assertEqual(instance['launched_at'], launched_at_dec) terminated_at_dec = stackutils.str_time_to_unix(terminated_at)