diff --git a/.gitignore b/.gitignore index a735f8b..6470873 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,8 @@ nosetests.xml # Translations *.mo + +# IDE Project Files +*.project +*.pydev* +*.idea diff --git a/bin/test-distiller.py b/bin/test-distiller.py index 0e38d3f..e9afcb0 100755 --- a/bin/test-distiller.py +++ b/bin/test-distiller.py @@ -3,8 +3,6 @@ # # Copyright © 2014 Rackspace Hosting. # -# Author: Monsyne Dragon -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -27,8 +25,8 @@ import argparse import json import sys -from stackdistiller import distiller from stackdistiller import condenser +from stackdistiller import distiller class TestCondenser(condenser.CondenserBase): @@ -81,28 +79,32 @@ def test_data(args): yield n -parser = argparse.ArgumentParser(description="Test Distiller configuration") -parser.add_argument('-c', '--config', - default='event_definitions.yaml', - help='Name of event definitions file ' - 'to test (Default: %(default)s)') -parser.add_argument('-l', '--list', action='store_true', - help='Test data files contain JSON list of notifications.' - ' (By default data files should contain a single ' - 'notification.)') -parser.add_argument('-d', '--add_default_definition', action='store_true', - help='Add default event definition. Normally, ' - 'notifications are dropped if there is no event ' - 'definition for their event_type. Setting this adds a ' - '"catchall" that converts unknown notifications to Events' - ' with a few basic traits.') -parser.add_argument('-o', '--output', type=argparse.FileType('w'), +parser = argparse.ArgumentParser( + description="Test Distiller configuration") +parser.add_argument( + '-c', '--config', + default='event_definitions.yaml', + help='Name of event definitions file ' + 'to test (Default: %(default)s)') +parser.add_argument( + '-l', '--list', action='store_true', + help='Test data files contain JSON list of notifications.' + ' (By default data files should contain a single ' + 'notification.)') +parser.add_argument( + '-d', '--add_default_definition', action='store_true', + help='Add default event definition. Normally, ' + 'notifications are dropped if there is no event ' + 'definition for their event_type. Setting this adds a ' + '"catchall" that converts unknown notifications to Events' + ' with a few basic traits.') +parser.add_argument('-o', '--output', type=argparse.FileType('w'), default=sys.stdout, help="Output file. Default stdout") -parser.add_argument('test_data', nargs='*', metavar='JSON_FILE', - help="Test notifications in JSON format. Defaults to stdin") +parser.add_argument( + 'test_data', nargs='*', metavar='JSON_FILE', + help="Test notifications in JSON format. Defaults to stdin") args = parser.parse_args() - config = distiller.load_config(args.config) out = args.output @@ -115,7 +117,7 @@ drops = 0 cond = TestCondenser() for notification in notifications: cond.clear() - nct +=1 + nct += 1 if dist.to_event(notification, cond) is None: out.write("Dropped notification: %s\n" % notification['message_id']) diff --git a/requirements.txt b/requirements.txt index d6e1198..482dd9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,6 @@ --e . +hacking>=0.10.0,<0.11 +enum34>=1.0 +iso8601>=0.1.10 +jsonpath-rw>=1.2.0, < 2.0 +PyYAML>=3.1.0 +six>=1.5.2 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..150dec5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,26 @@ +[metadata] +description-file = README.md +name = stackdistiller +version = 0.12 +author = Monsyne Dragon +author_email = mdragon@rackspace.com +summary = A data extraction and transformation library for OpenStack notifications +license = Apache-2 +keywords = + OpenStack + notifications + events + extraction + transformation +classifiers = + Development Status :: 3 - Alpha + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + +home-page = https://github.com/stackforge/stacktach-stackdistiller + +[files] +packages = + stackdistiller diff --git a/setup.py b/setup.py index 2247955..aa2d8a0 100644 --- a/setup.py +++ b/setup.py @@ -1,37 +1,8 @@ -import os -from setuptools import setup, find_packages +#!/usr/bin/env python - -def read(fname): - return open(os.path.join(os.path.dirname(__file__), fname)).read() +from setuptools import setup setup( - name='stackdistiller', - version='0.11', - author='Monsyne Dragon', - author_email='mdragon@rackspace.com', - description=("A data extraction and transformation library for " - "OpenStack notifications"), - license='Apache License (2.0)', - keywords='OpenStack notifications events extraction transformation', - packages=find_packages(exclude=['tests']), - classifiers=[ - 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - ], - url='https://github.com/stackforge/stacktach-stackdistiller', - scripts=['bin/test-distiller.py'], - long_description=read('README.md'), - install_requires=[ - "enum34 >= 1.0", - "iso8601 >= 0.1.10", - "jsonpath-rw >= 1.2.0, < 2.0", - "PyYAML >= 3.1.0", - "six >= 1.5.2", - ], - - zip_safe=False + setup_requires=['pbr'], + pbr=True, ) diff --git a/stackdistiller/condenser.py b/stackdistiller/condenser.py index eb5f7ad..99da164 100644 --- a/stackdistiller/condenser.py +++ b/stackdistiller/condenser.py @@ -2,8 +2,6 @@ # # Copyright © 2014 Rackspace Hosting. # -# Author: Monsyne Dragon -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -22,18 +20,23 @@ import six @six.add_metaclass(abc.ABCMeta) class CondenserBase(object): - """Base class for Condenser objects that collect data extracted from a - Notification by the Distiller, and format it into a usefull datastructure. + """Base class for Condenser objects - A simple Condenser may just colect all the traits received into a dictionary. - More complex ones may build collections of application or ORM model objects, - or XML document trees. + Collect data extracted from a Notification by the Distiller, and + format it into a useful data structure. - Condensers also have hooks for verification logic, to check that all needed - traits are present.""" + A simple Condenser may just colect all the traits received into + a dictionary. More complex ones may build collections of application + or ORM model objects, or XML document trees. + + Condensers also have hooks for verification logic, to check that + all needed traits are present. + """ def __init__(self, **kw): - """Setup the condenser. A new instance of the condenser is passed to the + """Set up the condenser. + + A new instance of the condenser is passed to the distiller for each notification extracted. :param kw: keyword parameters for condenser. @@ -43,7 +46,9 @@ class CondenserBase(object): @abc.abstractmethod def add_trait(self, name, trait_type, value): - """Add a trait to the Event datastructure being built by this + """Add a trait + + Add a trait to the Event data structure being built by this condenser. The distiller will call this for each extracted trait. :param name: (string) name of the trait @@ -54,7 +59,9 @@ class CondenserBase(object): @abc.abstractmethod def add_envelope_info(self, event_type, message_id, when): - """Add the metadata for this event, extracted from the notification's + """Add the metadata for this event + + Add metadata extracted from the notification's envelope. The distiller will call this once. :param event_type: (string) Type of event, as a dotted string such as @@ -66,14 +73,14 @@ class CondenserBase(object): @abc.abstractmethod def get_event(self): - """Return the Event datastructure constructed by this condenser.""" + """Return the Event data structure constructed by this condenser.""" @abc.abstractmethod def clear(self): """Clear condenser state.""" def validate(self): - """Check Event against whatever validation logic this condenser may have + """Check Event against whatever validation logic this condenser has :returns: (bool) True if valid. @@ -83,6 +90,7 @@ class CondenserBase(object): class DictionaryCondenser(CondenserBase): """Return event data as a simple python dictionary""" + def __init__(self, **kw): self.clear() super(DictionaryCondenser, self).__init__(**kw) diff --git a/stackdistiller/distiller.py b/stackdistiller/distiller.py index b8b6c20..fa84148 100644 --- a/stackdistiller/distiller.py +++ b/stackdistiller/distiller.py @@ -2,8 +2,6 @@ # # Copyright © 2013 Rackspace Hosting. # -# Author: Monsyne Dragon -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -20,7 +18,6 @@ import collections import datetime import fnmatch import logging -import os from enum import Enum import iso8601 @@ -60,13 +57,13 @@ def load_config(filename): if hasattr(err, 'problem_mark'): mark = err.problem_mark errmsg = ("Invalid YAML syntax in Event Definitions file " - "%(file)s at line: %(line)s, column: %(column)s." + "%(file)s at line: %(line)s, column: %(column)s." % dict(file=filename, line=mark.line + 1, column=mark.column + 1)) else: errmsg = ("YAML error reading Event Definitions file " - "%(file)s" + "%(file)s" % dict(file=filename)) logger.error(errmsg) raise @@ -100,7 +97,6 @@ Trait = collections.namedtuple('Trait', ('name', 'trait_type', 'value')) class TraitDefinition(object): - def __init__(self, name, trait_cfg, plugin_map): self.cfg = trait_cfg self.name = name @@ -110,8 +106,8 @@ class TraitDefinition(object): type_name = trait_cfg.get('type', 'text') except AttributeError as e: raise EventDefinitionException( - "Unable to get type for '%s'" % trait_cfg, - self.cfg) + "Unable to get type for '%s'" % trait_cfg, + self.cfg) if 'plugin' in trait_cfg: plugin_cfg = trait_cfg['plugin'] @@ -124,7 +120,7 @@ class TraitDefinition(object): except KeyError: raise EventDefinitionException( 'Plugin specified, but no plugin name supplied for ' - 'trait %s' % name, self.cfg) + 'trait %s' % name, self.cfg) plugin_params = plugin_cfg.get('parameters') if plugin_params is None: plugin_params = {} @@ -133,8 +129,8 @@ class TraitDefinition(object): except KeyError: raise EventDefinitionException( 'No plugin named %(plugin)s available for ' - 'trait %(trait)s' % dict(plugin=plugin_name, - trait=name), self.cfg) + 'trait %(trait)s' % dict(plugin=plugin_name, + trait=name), self.cfg) self.plugin = plugin_class(**plugin_params) else: self.plugin = None @@ -142,7 +138,7 @@ class TraitDefinition(object): if 'fields' not in trait_cfg: raise EventDefinitionException( "Required field in trait definition not specified: " - "'%s'" % 'fields', + "'%s'" % 'fields', self.cfg) fields = trait_cfg['fields'] @@ -157,7 +153,7 @@ class TraitDefinition(object): except Exception as e: raise EventDefinitionException( "Parse error in JSONPath specification " - "'%(jsonpath)s' for %(trait)s: %(err)s" + "'%(jsonpath)s' for %(trait)s: %(err)s" % dict(jsonpath=fields, trait=name, err=e), self.cfg) try: self.trait_type = Datatype[type_name] @@ -196,7 +192,6 @@ class TraitDefinition(object): class EventDefinition(object): - DEFAULT_TRAITS = dict( service=dict(type='text', fields='publisher_id'), request_id=dict(type='text', fields='_context_request_id'), @@ -262,8 +257,8 @@ class EventDefinition(object): @staticmethod def _extract_when(body): - """Extract the generated datetime from the notification. - """ + """Extract the generated datetime from the notification.""" + # NOTE: I am keeping the logic the same as it was in openstack # code, However, *ALL* notifications should have a 'timestamp' # field, it's part of the notification envelope spec. If this was diff --git a/stackdistiller/trait_plugins.py b/stackdistiller/trait_plugins.py index 0b8b96e..e970979 100644 --- a/stackdistiller/trait_plugins.py +++ b/stackdistiller/trait_plugins.py @@ -2,8 +2,6 @@ # # Copyright © 2013 Rackspace Hosting. # -# Author: Monsyne Dragon -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -22,9 +20,7 @@ import six @six.add_metaclass(abc.ABCMeta) class TraitPluginBase(object): - """Base class for plugins that convert notification fields to - Trait values. - """ + """Base class for plugins that convert notification fields to Traits""" def __init__(self, **kw): """Setup the trait plugin. diff --git a/tests/test_distiller.py b/tests/test_distiller.py index 64fd87d..6be8902 100644 --- a/tests/test_distiller.py +++ b/tests/test_distiller.py @@ -2,13 +2,11 @@ # # Copyright © 2013 Rackspace Hosting. # -# Author: Monsyne Dragon -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT @@ -18,7 +16,7 @@ import datetime -#for Python2.6 compatability. +# for Python2.6 compatability. import unittest2 as unittest import iso8601 @@ -50,13 +48,14 @@ class TestCondenser(object): class DistillerTestBase(unittest.TestCase): def _create_test_notification(self, event_type, message_id, **kw): - return dict(event_type=event_type, - message_id=message_id, - priority="INFO", - publisher_id="compute.host-1-2-3", - timestamp="2013-08-08 21:06:37.803826", - payload=kw, - ) + return dict( + event_type=event_type, + message_id=message_id, + priority="INFO", + publisher_id="compute.host-1-2-3", + timestamp="2013-08-08 21:06:37.803826", + payload=kw, + ) def assertIsValidEvent(self, event, notification): self.assertIsNot( @@ -113,7 +112,6 @@ class DistillerTestBase(unittest.TestCase): class TestTraitDefinition(DistillerTestBase): - def setUp(self): super(TestTraitDefinition, self).setUp() self.n1 = self._create_test_notification( @@ -126,8 +124,8 @@ class TestTraitDefinition(DistillerTestBase): host='host-1-2-3', bogus_date='', image_meta=dict( - disk_gb='20', - thing='whatzit'), + disk_gb='20', + thing='whatzit'), foobar=50) self.test_plugin_class = mock.MagicMock(name='mock_test_plugin') @@ -244,14 +242,14 @@ class TestTraitDefinition(DistillerTestBase): def test_to_trait_multiple_different_nesting(self): cfg = dict(type='int', fields=['payload.foobar', - 'payload.image_meta.disk_gb']) + 'payload.image_meta.disk_gb']) tdef = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) t = tdef.to_trait(self.n1) self.assertEqual(50, t.value) cfg = dict(type='int', fields=['payload.image_meta.disk_gb', - 'payload.foobar']) + 'payload.foobar']) tdef = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) t = tdef.to_trait(self.n1) @@ -322,7 +320,7 @@ class TestTraitDefinition(DistillerTestBase): jsonpath_rw.parse('(payload.test)|(payload.other)')) def test_invalid_path_config(self): - #test invalid jsonpath... + # test invalid jsonpath... cfg = dict(fields='payload.bogus(') self.assertRaises(distiller.EventDefinitionException, distiller.TraitDefinition, @@ -331,7 +329,7 @@ class TestTraitDefinition(DistillerTestBase): self.fake_plugin_map) def test_invalid_plugin_config(self): - #test invalid jsonpath... + # test invalid jsonpath... cfg = dict(fields='payload.test', plugin=dict(bogus="true")) self.assertRaises(distiller.EventDefinitionException, distiller.TraitDefinition, @@ -340,7 +338,7 @@ class TestTraitDefinition(DistillerTestBase): self.fake_plugin_map) def test_unknown_plugin(self): - #test invalid jsonpath... + # test invalid jsonpath... cfg = dict(fields='payload.test', plugin=dict(name='bogus')) self.assertRaises(distiller.EventDefinitionException, distiller.TraitDefinition, @@ -366,7 +364,7 @@ class TestTraitDefinition(DistillerTestBase): self.assertEqual(distiller.Datatype.datetime, t.trait_type) def test_invalid_type_config(self): - #test invalid jsonpath... + # test invalid jsonpath... cfg = dict(type='bogus', fields='payload.test') self.assertRaises(distiller.EventDefinitionException, distiller.TraitDefinition, @@ -376,7 +374,6 @@ class TestTraitDefinition(DistillerTestBase): class TestEventDefinition(DistillerTestBase): - def setUp(self): super(TestEventDefinition, self).setUp() @@ -419,11 +416,13 @@ class TestEventDefinition(DistillerTestBase): e = edef.to_event(self.test_notification1, self.condenser) self.assertTrue(e is self.condenser) self.assertEqual('test.thing', e.event_type) - self.assertEqual(datetime.datetime(2013, 8, 8, 21, 6, 37, 803826, iso8601.iso8601.UTC), + self.assertEqual(datetime.datetime(2013, 8, 8, 21, 6, 37, 803826, + iso8601.iso8601.UTC), e.when) self.assertHasDefaultTraits(e) - self.assertHasTrait(e, 'host', value='host-1-2-3', trait_type=trait_type) + self.assertHasTrait(e, 'host', value='host-1-2-3', + trait_type=trait_type) self.assertHasTrait(e, 'instance_id', value='uuid-for-instance-0001', trait_type=trait_type) @@ -609,24 +608,25 @@ class TestEventDefinition(DistillerTestBase): class TestDistiller(DistillerTestBase): - def setUp(self): super(TestDistiller, self).setUp() - self.valid_event_def1 = [{ - 'event_type': 'compute.instance.create.*', - 'traits': { - 'instance_id': { - 'type': 'text', - 'fields': ['payload.instance_uuid', - 'payload.instance_id'], + self.valid_event_def1 = [ + { + 'event_type': 'compute.instance.create.*', + 'traits': { + 'instance_id': { + 'type': 'text', + 'fields': ['payload.instance_uuid', + 'payload.instance_id'], + }, + 'host': { + 'type': 'text', + 'fields': 'payload.host', + }, }, - 'host': { - 'type': 'text', - 'fields': 'payload.host', - }, - }, - }] + } + ] self.test_notification1 = self._create_test_notification( "compute.instance.create.start", @@ -645,10 +645,7 @@ class TestDistiller(DistillerTestBase): # test a malformed notification now = datetime.datetime.utcnow().replace(tzinfo=iso8601.iso8601.UTC) mock_utcnow.return_value = now - c = distiller.Distiller( - [], - self.fake_plugin_map, - catchall=True) + c = distiller.Distiller([], self.fake_plugin_map, catchall=True) message = {'event_type': "foo", 'message_id': "abc", 'publisher_id': "1"} @@ -674,7 +671,8 @@ class TestDistiller(DistillerTestBase): e = c.to_event(self.test_notification2, TestCondenser()) self.assertIsValidEvent(e, self.test_notification2) self.assertEqual(1, len(e.traits), - "Wrong number of traits %s: %s" % (len(e.traits), e.traits)) + "Wrong number of traits %s: %s" % ( + len(e.traits), e.traits)) self.assertHasDefaultTraits(e) self.assertDoesNotHaveTrait(e, 'instance_id') self.assertDoesNotHaveTrait(e, 'host') @@ -696,10 +694,7 @@ class TestDistiller(DistillerTestBase): self.assertIsNotValidEvent(e, self.test_notification2) def test_distiller_empty_cfg_with_catchall(self): - c = distiller.Distiller( - [], - self.fake_plugin_map, - catchall=True) + c = distiller.Distiller([], self.fake_plugin_map, catchall=True) self.assertEqual(1, len(c.definitions)) e = c.to_event(self.test_notification1, TestCondenser()) self.assertIsValidEvent(e, self.test_notification1) @@ -712,10 +707,7 @@ class TestDistiller(DistillerTestBase): self.assertHasDefaultTraits(e) def test_distiller_empty_cfg_without_catchall(self): - c = distiller.Distiller( - [], - self.fake_plugin_map, - catchall=False) + c = distiller.Distiller([], self.fake_plugin_map, catchall=False) self.assertEqual(0, len(c.definitions)) e = c.to_event(self.test_notification1, TestCondenser()) self.assertIsNotValidEvent(e, self.test_notification1) diff --git a/tests/test_trait_plugins.py b/tests/test_trait_plugins.py index d0c386a..c83200e 100644 --- a/tests/test_trait_plugins.py +++ b/tests/test_trait_plugins.py @@ -2,8 +2,6 @@ # # Copyright © 2013 Rackspace Hosting. # -# Author: Monsyne Dragon -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -16,14 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -#for Python2.6 compatability. +# for Python2.6 compatability. import unittest2 as unittest from stackdistiller import trait_plugins class TestSplitterPlugin(unittest.TestCase): - def setUp(self): super(TestSplitterPlugin, self).setUp() self.pclass = trait_plugins.SplitterTraitPlugin @@ -70,7 +67,6 @@ class TestSplitterPlugin(unittest.TestCase): class TestBitfieldPlugin(unittest.TestCase): - def setUp(self): super(TestBitfieldPlugin, self).setUp() self.pclass = trait_plugins.BitfieldTraitPlugin diff --git a/tox.ini b/tox.ini index 9afa955..434241a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27 +envlist = py26,py27,pep8 [testenv] deps = @@ -13,3 +13,11 @@ commands = sitepackages = False +[testenv:pep8] +commands = + flake8 + +[flake8] +ignore = +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg +show-source = True