diff --git a/distil/collector/ceilometer.py b/distil/collector/ceilometer.py index 97aae2f..b470e80 100644 --- a/distil/collector/ceilometer.py +++ b/distil/collector/ceilometer.py @@ -66,4 +66,8 @@ class CeilometerCollector(base.BaseCollector): sample_objs = self.cclient.new_samples.list(q=query) + # The samples are in descending order by default, should change it to + # be ascending, making the logic consistent with deprecated code. + sample_objs.reverse() + return [obj.to_dict() for obj in sample_objs] diff --git a/distil/common/openstack.py b/distil/common/openstack.py index ce02384..1389342 100644 --- a/distil/common/openstack.py +++ b/distil/common/openstack.py @@ -147,12 +147,3 @@ def get_volume_type(volume_type): return vtype['name'] return None - - -@general.disable_ssl_warnings -def get_flavor_name(flavor_id): - """Grabs the correct flavor name from Nova given the correct ID.""" - if flavor_id not in cache: - nova = get_nova_client() - cache[flavor_id] = nova.flavors.get(flavor_id).name - return cache[flavor_id] diff --git a/distil/tests/unit/transformer/__init__.py b/distil/tests/unit/transformer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/distil/tests/unit/transformer/test_arithmetic.py b/distil/tests/unit/transformer/test_arithmetic.py new file mode 100644 index 0000000..18ead90 --- /dev/null +++ b/distil/tests/unit/transformer/test_arithmetic.py @@ -0,0 +1,250 @@ +# Copyright (C) 2017 Catalyst IT Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +import mock + +from distil.common.constants import date_format +from distil.common import general +from distil.tests.unit import base +from distil.transformer import arithmetic + +p = lambda t: datetime.datetime.strptime(t, date_format) + + +class FAKE_DATA: + t0 = p('2014-01-01T00:00:00') + t0_10 = p('2014-01-01T00:10:00') + t0_20 = p('2014-01-01T00:30:00') + t0_30 = p('2014-01-01T00:30:00') + t0_40 = p('2014-01-01T00:40:00') + t0_50 = p('2014-01-01T00:50:00') + t1 = p('2014-01-01T01:00:00') + + +@mock.patch.object(general, 'get_transformer_config', mock.Mock()) +class TestMaxTransformer(base.DistilTestCase): + def test_all_different_values(self): + """ + Tests that the transformer correctly grabs the highest value, + when all values are different. + """ + + data = [ + {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': 12}, + {'timestamp': FAKE_DATA.t0_10.isoformat(), 'volume': 3}, + {'timestamp': FAKE_DATA.t0_20.isoformat(), 'volume': 7}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), 'volume': 3}, + {'timestamp': FAKE_DATA.t0_40.isoformat(), 'volume': 25}, + {'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 2}, + {'timestamp': FAKE_DATA.t1.isoformat(), 'volume': 6}, + ] + + xform = arithmetic.MaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 25}, usage) + + def test_all_same_values(self): + """ + Tests that that transformer correctly grabs any value, + when all values are the same. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': 25}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 25}, + {'timestamp': FAKE_DATA.t1, 'volume': 25}, + ] + + xform = arithmetic.MaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 25}, usage) + + def test_none_value(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': None}, + ] + + xform = arithmetic.MaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 0}, usage) + + def test_none_and_other_values(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': None}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 25}, + {'timestamp': FAKE_DATA.t1, 'volume': 27}, + ] + + xform = arithmetic.MaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 27}, usage) + + +@mock.patch.object(general, 'get_transformer_config', mock.Mock()) +class TestStorageMaxTransformer(base.DistilTestCase): + def test_all_different_values(self): + """ + Tests that the transformer correctly grabs the highest value, + when all values are different. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': 12, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_10, 'volume': 3, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_20, 'volume': 7, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 3, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_40, 'volume': 25, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_50, 'volume': 2, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t1, 'volume': 6, + 'metadata': {}}, + ] + + xform = arithmetic.StorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 25}, usage) + + def test_all_same_values(self): + """ + Tests that that transformer correctly grabs any value, + when all values are the same. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': 25, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 25, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t1, 'volume': 25, + 'metadata': {}}, + ] + + xform = arithmetic.StorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 25}, usage) + + def test_none_value(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': None, + 'metadata': {}}, + ] + + xform = arithmetic.StorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 0}, usage) + + def test_none_and_other_values(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0, 'volume': None, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t0_30, 'volume': 25, + 'metadata': {}}, + {'timestamp': FAKE_DATA.t1, 'volume': 27, + 'metadata': {}}, + ] + + xform = arithmetic.StorageMaxTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 27}, usage) + + +@mock.patch.object(general, 'get_transformer_config', mock.Mock()) +class TestSumTransformer(base.DistilTestCase): + def test_basic_sum(self): + """ + Tests that the transformer correctly calculate the sum value. + """ + + data = [ + {'timestamp': '2014-01-01T00:00:00', 'volume': 1}, + {'timestamp': '2014-01-01T00:10:00', 'volume': 1}, + {'timestamp': '2014-01-01T01:00:00', 'volume': 1}, + ] + + xform = arithmetic.SumTransformer() + usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'fake_meter': 2}, usage) + + def test_none_value(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None}, + ] + + xform = arithmetic.SumTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 0}, usage) + + def test_none_and_other_values(self): + """ + Tests that that transformer correctly handles a None value. + """ + + data = [ + {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), 'volume': 25}, + {'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 25}, + ] + + xform = arithmetic.SumTransformer() + usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'some_meter': 50}, usage) diff --git a/distil/tests/unit/transformer/test_conversion.py b/distil/tests/unit/transformer/test_conversion.py new file mode 100644 index 0000000..ef3a728 --- /dev/null +++ b/distil/tests/unit/transformer/test_conversion.py @@ -0,0 +1,374 @@ +# Copyright (C) 2017 Catalyst IT Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import datetime + +import mock + +from distil.common.constants import date_format +from distil.common import general +from distil.tests.unit import base +from distil.transformer import conversion + +p = lambda t: datetime.datetime.strptime(t, date_format) + + +class FAKE_DATA: + t0 = p('2014-01-01T00:00:00') + t0_10 = p('2014-01-01T00:10:00') + t0_30 = p('2014-01-01T00:30:00') + t1 = p('2014-01-01T01:00:00') + + # and one outside the window + tpre = p('2013-12-31T23:50:00') + + flavor = '1' + flavor2 = '2' + + +FAKE_CONFIG = { + "uptime": { + "tracked_states": ["active", "building", "paused", "rescued", + "resized"] + }, + "from_image": { + "service": "volume.size", + "md_keys": ["image_ref", "image_meta.base_image_ref"], + "none_values": ["None", ""], + "size_keys": ["root_gb"] + } +} + + +@mock.patch.object(general, 'get_transformer_config', + mock.Mock(return_value=FAKE_CONFIG)) +class TestUpTimeTransformer(base.DistilTestCase): + def test_trivial_run(self): + """ + Test that an no input data produces empty uptime. + """ + state = [] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({}, result) + + def test_online_constant_flavor(self): + """ + Test that a machine online for a 1h period with constant + flavor works and gives 1h of uptime. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({FAKE_DATA.flavor: 3600}, result) + + def test_offline_constant_flavor(self): + """ + Test that a machine offline for a 1h period with constant flavor + works and gives zero uptime. + """ + + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'stopped'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'stopped'}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({}, result) + + def test_shutdown_during_period(self): + """ + Test that a machine run for 0.5 then shutdown gives 0.5h uptime. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'stopped'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'stopped'}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({FAKE_DATA.flavor: 1800}, result) + + def test_online_flavor_change(self): + """ + Test that a machine run for 0.5h as m1.tiny, resized to m1.large, + and run for a further 0.5 yields 0.5h of uptime in each class. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor2, + 'status': 'active'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor2, + 'status': 'active'}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertDictEqual( + {FAKE_DATA.flavor: 1800, FAKE_DATA.flavor2: 1800}, + result + ) + + def test_period_leadin_none_available(self): + """ + Test that if the first data point is well into the window, and we had + no lead-in data, we assume no usage until our first real data point. + """ + state = [ + {'timestamp': FAKE_DATA.t0_10.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({FAKE_DATA.flavor: 3000}, result) + + def test_period_leadin_available(self): + """ + Test that if the first data point is well into the window, but we *do* + have lead-in data, then we use the lead-in clipped to the start of the + window. + """ + state = [ + {'timestamp': FAKE_DATA.tpre.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}}, + {'timestamp': FAKE_DATA.t0_10.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'status': 'active'}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({FAKE_DATA.flavor: 3600}, result) + + def test_notification_case(self): + """ + Test that the transformer handles the notification metedata key, + if/when it can't find the status key. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'state': 'active'}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor, + 'state': 'active'}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({FAKE_DATA.flavor: 3600}, result) + + def test_no_state_in_metedata(self): + """ + Test that the transformer doesn't fall over if there isn't one of + the two state/status key options in the metadata. + """ + state = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'instance_type': FAKE_DATA.flavor}} + ] + + xform = conversion.UpTimeTransformer() + result = xform.transform_usage('state', state, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({}, result) + + +@mock.patch.object(general, 'get_transformer_config', + mock.Mock(return_value=FAKE_CONFIG)) +class TestFromImageTransformer(base.DistilTestCase): + """ + These tests rely on config settings for from_image, + as defined in test constants, or in conf.yaml + """ + + def test_from_volume_case(self): + """ + If instance is booted from volume transformer should return none. + """ + data = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'image_ref': ""}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'image_ref': "None"}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'image_ref': "None"}} + ] + + data2 = [ + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'image_ref': "None"}} + ] + + xform = conversion.FromImageTransformer() + + usage = xform.transform_usage('instance', data, FAKE_DATA.t0, + FAKE_DATA.t1) + usage2 = xform.transform_usage('instance', data2, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertIsNone(usage) + self.assertIsNone(usage2) + + def test_default_to_from_volume_case(self): + """ + Unless all image refs contain something, assume booted from volume. + """ + data = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'image_ref': ""}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'image_ref': "d5a4f118023928195f4ef"}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'image_ref': "None"}} + ] + + xform = conversion.FromImageTransformer() + usage = xform.transform_usage('instance', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertIsNone(usage) + + def test_from_image_case(self): + """ + If all image refs contain something, should return entry. + """ + data = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'image_ref': "d5a4f118023928195f4ef", + 'root_gb': "20"}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'image_ref': "d5a4f118023928195f4ef", + 'root_gb': "20"}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'image_ref': "d5a4f118023928195f4ef", + 'root_gb': "20"}} + ] + + xform = conversion.FromImageTransformer() + usage = xform.transform_usage('instance', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'volume.size': 20}, usage) + + def test_from_image_case_highest_size(self): + """ + If all image refs contain something, + should return entry with highest size from data. + """ + data = [ + {'timestamp': FAKE_DATA.t0.isoformat(), + 'metadata': {'image_ref': "d5a4f118023928195f4ef", + 'root_gb': "20"}}, + {'timestamp': FAKE_DATA.t0_30.isoformat(), + 'metadata': {'image_ref': "d5a4f118023928195f4ef", + 'root_gb': "60"}}, + {'timestamp': FAKE_DATA.t1.isoformat(), + 'metadata': {'image_ref': "d5a4f118023928195f4ef", + 'root_gb': "20"}} + ] + + xform = conversion.FromImageTransformer() + usage = xform.transform_usage('instance', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'volume.size': 60}, usage) + + +@mock.patch.object(general, 'get_transformer_config', + mock.Mock(return_value=FAKE_CONFIG)) +class TestNetworkServiceTransformer(base.DistilTestCase): + def test_basic_sum(self): + """Tests that the transformer correctly calculate the sum value. + """ + + data = [ + {'timestamp': '2014-01-01T00:00:00', 'volume': 1}, + {'timestamp': '2014-01-01T00:10:00', 'volume': 0}, + {'timestamp': '2014-01-01T01:00:00', 'volume': 2}, + ] + + xform = conversion.NetworkServiceTransformer() + usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'fake_meter': 1}, usage) + + def test_only_pending_service(self): + """Tests that the transformer correctly calculate the sum value. + """ + + data = [ + {'timestamp': '2014-01-01T00:00:00', 'volume': 2}, + {'timestamp': '2014-01-01T00:10:00', 'volume': 2}, + {'timestamp': '2014-01-01T01:00:00', 'volume': 2}, + ] + + xform = conversion.NetworkServiceTransformer() + usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0, + FAKE_DATA.t1) + + self.assertEqual({'fake_meter': 0}, usage) diff --git a/distil/transformer/arithmetic.py b/distil/transformer/arithmetic.py index ea4f025..169e590 100644 --- a/distil/transformer/arithmetic.py +++ b/distil/transformer/arithmetic.py @@ -30,9 +30,11 @@ class MaxTransformer(BaseTransformer): """ def _transform_usage(self, meter_name, raw_data, start_at, end_at): - max_vol = max([v["volume"] - for v in raw_data]) if len(raw_data) else 0 + max_vol = max([v["volume"] for v in raw_data]) if len(raw_data) else 0 + max_vol = max_vol if max_vol else 0 + hours = (end_at - start_at).total_seconds() / 3600.0 + return {meter_name: max_vol * hours} @@ -47,13 +49,7 @@ class StorageMaxTransformer(BaseTransformer): if not data: return None - max_vol = max([v["volume"] for v in data]) - - if max_vol is None: - max_vol = 0 - LOG.warning("None max_vol value for %s in window: %s - %s " % - (name, start.strftime(constants.iso_time), - end.strftime(constants.iso_time))) + max_vol = max([v["volume"] for v in data]) or 0 if "volume_type" in data[-1]['metadata']: vtype = data[-1]['metadata']['volume_type'] @@ -83,5 +79,6 @@ class SumTransformer(BaseTransformer): '%Y-%m-%dT%H:%M:%S') if t >= start_at and t < end_at: - sum_vol += sample["volume"] + sum_vol += sample["volume"] or 0 + return {meter_name: sum_vol} diff --git a/distil/transformer/conversion.py b/distil/transformer/conversion.py index 8a54473..9ba4e14 100644 --- a/distil/transformer/conversion.py +++ b/distil/transformer/conversion.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -import datetime +from datetime import datetime from distil.transformer import BaseTransformer from distil.common import constants @@ -29,14 +29,12 @@ class UpTimeTransformer(BaseTransformer): # get tracked states from config tracked = self.config['uptime']['tracked_states'] - tracked_states = {constants.states[i] for i in tracked} - usage_dict = {} def sort_and_clip_end(usage): cleaned = (self._clean_entry(s) for s in usage) - clipped = (s for s in cleaned if s['timestamp'] < end) - return sorted(clipped, key=lambda x: x['timestamp']) + clipped = [s for s in cleaned if s['timestamp'] < end] + return clipped state = sort_and_clip_end(data) @@ -57,7 +55,7 @@ class UpTimeTransformer(BaseTransformer): usage_dict[flav] = usage_dict.get(flav, 0) + diff.total_seconds() for val in state[1:]: - if last_state["volume"] in tracked_states: + if last_state["status"] in tracked: diff = val["timestamp"] - last_timestamp if val['timestamp'] > last_timestamp: # if diff < 0 then we were looking back before the start @@ -70,29 +68,25 @@ class UpTimeTransformer(BaseTransformer): # extend the last state we know about, to the end of the window, # if we saw any actual uptime. - if (end and last_state['volume'] in tracked_states - and seen_sample_in_window): + if (end and last_state['status'] in tracked and seen_sample_in_window): diff = end - last_timestamp _add_usage(diff) - # map the flavors to names on the way out - return {openstack.get_flavor_name(f): v for f, v in usage_dict.items()} + return usage_dict def _clean_entry(self, entry): result = { - 'volume': entry['volume'], - 'flavor': entry['metadata'].get( - 'flavor.id', entry['metadata'].get( - 'instance_flavor_id', 0 + 'status': entry['metadata'].get( + 'status', entry['metadata'].get( + 'state', "" ) + ), + 'flavor': entry['metadata'].get('instance_type'), + 'timestamp': datetime.strptime( + entry['timestamp'], + "%Y-%m-%dT%H:%M:%S" ) } - try: - result['timestamp'] = datetime.datetime.strptime( - entry['timestamp'], constants.date_format) - except ValueError: - result['timestamp'] = datetime.datetime.strptime( - entry['timestamp'], constants.date_format_f) return result @@ -126,7 +120,9 @@ class FromImageTransformer(BaseTransformer): size = root_size except KeyError: pass + hours = (end - start).total_seconds() / 3600.0 + return {service: size * hours} @@ -141,7 +137,8 @@ class NetworkServiceTransformer(BaseTransformer): # blob/master/ceilometer/network/services/vpnaas.py#L55), so we have # to check the volume to make sure only the active service is # charged(0=inactive, 1=active). - max_vol = max([v["volume"] for v in data - if v["volume"] < 2]) if len(data) else 0 + volumes = [v["volume"] for v in data if + v["volume"] < 2] + max_vol = max(volumes) if len(volumes) else 0 hours = (end - start).total_seconds() / 3600.0 return {name: max_vol * hours} diff --git a/etc/distil.conf.sample b/etc/distil.conf.sample deleted file mode 100644 index 8b8cf75..0000000 --- a/etc/distil.conf.sample +++ /dev/null @@ -1,41 +0,0 @@ -[DEFAULT] -debug = True -ignore_tenants = demo -timezone = Pacific/Auckland -host = localhost -port = 9999 - -[collector] -periodic_interval = 10 -dawn_of_time = 2016-05-18 01:00:00 -meter_mappings_file = /etc/distil/meter_mappings.yml - -[rater] -rater_type = odoo -rate_file_path = /etc/distil/rates.csv - -[odoo] -version=8.0 -hostname= -port=443 -protocol=jsonrpc+ssl -database= -user= -password= - -[database] -connection = mysql://root:passw0rd@127.0.0.1/distil?charset=utf8 -backend = sqlalchemy - -[keystone_authtoken] -memcache_servers = 127.0.0.1:11211 -signing_dir = /var/cache/distil -cafile = /opt/stack/data/ca-bundle.pem -auth_uri = http://127.0.0.1:5000 -project_domain_id = default -project_name = service -user_domain_id = default -password = passw0rd -username = distil -auth_url = http://127.0.0.1:35357 -auth_type = password \ No newline at end of file diff --git a/etc/transformer.yaml b/etc/transformer.yaml deleted file mode 100644 index 026306f..0000000 --- a/etc/transformer.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# configuration for defining usage collection -collection: - # defines which meter is mapped to which transformer - meter_mappings: - # meter name as seen in ceilometer - state: - # type of resource it maps to (seen on sales order) - type: Virtual Machine - # which transformer to use - transformer: uptime - # what unit type is coming in via the meter - unit: second - metadata: - name: - sources: - # which keys to search for in the ceilometer entry metadata - # this can be more than one as metadata is inconsistent between - # source types - - display_name - availability zone: - sources: - - OS-EXT-AZ:availability_zone - ip.floating: - type: Floating IP - transformer: max - unit: hour - metadata: - ip address: - sources: - - floating_ip_address - volume.size: - type: Volume - transformer: max - unit: gigabyte - metadata: - name: - sources: - - display_name - availability zone: - sources: - - availability_zone - instance: - type: Volume - transformer: fromimage - unit: gigabyte - # if true allows id pattern, and metadata patterns - transform_info: True - # allows us to put the id into a pattern, - # only if transform_info is true, - # such as to append something to it - res_id_template: "%s-root_disk" - metadata: - name: - sources: - - display_name - template: "%s - root disk" - availability zone: - sources: - - availability_zone - image.size: - type: Image - transformer: max - unit: byte - metadata: - name: - sources: - - name - - properties.image_name - bandwidth: - type: Network Traffic - transformer: sum - unit: byte - metadata: - meter_label_id: - sources: - - label_id - network.services.vpn: - type: VPN - transformer: networkservice - unit: hour - metadata: - name: - sources: - - name - subnet: - sources: - - subnet_id - network: - type: Network - transformer: max - unit: hour - metadata: - name: - sources: - - name -# transformer configs -transformers: - uptime: - # states marked as "billable" for VMs. - tracked_states: - - active - - paused - - rescued - - resized - from_image: - service: volume.size - # What metadata values to check - md_keys: - - image_ref - - image_meta.base_image_ref - none_values: - - None - - "" - # where to get volume size from - size_keys: - - root_gb diff --git a/etc/transformer.yaml.sample b/etc/transformer.yaml.sample new file mode 100644 index 0000000..1a9eaa9 --- /dev/null +++ b/etc/transformer.yaml.sample @@ -0,0 +1,26 @@ +uptime: + # states marked as "billable" for VMs. + tracked_states: + - active + - paused + - rescue + - rescued + - resize + - resized + - verify_resize + - suspended + # uncomment these when we want to bill shutdown: + # - shutoff + # - stopped +from_image: + service: b1.standard + # What metadata values to check + md_keys: + - image_ref + - image_meta.base_image_ref + none_values: + - None + - "" + # where to get volume size from + size_keys: + - root_gb