Initial Commit

.gitignore

@@ -0,0 +1,18 @@
+AUTHORS
+ChangeLog
+*.pyc
+*.log
+*.swp
+*.swo
+*.egg*
+.tox
+.venv
+dist
+build
+.testrepository
+.idea
+.project
+.pydevproject
+.coverage*
+!.coveragerc
+cover/

.testr.conf

@@ -0,0 +1,8 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \
+             OS_TEST_LOCK_PATH=${OS_TEST_LOCK_PATH:-${TMPDIR:-'/tmp'}} \
+             ${PYTHON:-python} -m subunit.run discover -t ./ ./lpmqtt/tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list

CONTRIBUTING.rst

@@ -0,0 +1,15 @@
+============
+Contributing
+============
+
+If you would like to contribute to the development of OpenStack,
+you must follow the steps in this page:
+
+   http://docs.openstack.org/infra/manual/developers.html
+
+If you already have a good understanding of how the system works and your
+OpenStack accounts are set up, you can skip to the development workflow section
+of this documentation to learn how changes to OpenStack should be submitted for
+review via the Gerrit tool:
+
+   http://docs.openstack.org/infra/manual/developers.html#development-workflow

LICENSE

@@ -0,0 +1,175 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.

README.rst

@@ -0,0 +1,20 @@
+======
+lpmqtt
+======
+
+lpmqtt, like its name implies, is a tool for publish a launchpad event stream
+into MQTT. It will publish all the capture events from the lp emails over imap
+and publish them to MQTT
+
+MQTT Topics
+===========
+lpmqtt will push gerrit events to topics broken by project and event type.
+The formula used is::
+
+  <base topic>/<bugs or blueprints>/<number>
+
+
+Configuration
+=============
+There are a few required pieces of information to make lpmqtt work properly.
+These settings are specified in the config file.

lpmqtt/daemon.py

@@ -0,0 +1,104 @@
+# Copyright 2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import sys
+
+import paho.mqtt.publish as publish
+from six.moves import configparser
+
+from lpmqtt import lp
+
+
+class PushMQTT(object):
+    def __init__(self, hostname, port=1883, client_id=None,
+                 keepalive=60, will=None, auth=None, tls=None):
+        self.hostname = hostname
+        self.port = port
+        self.client_id = client_id
+        self.keepalive = 60
+        self.will = will
+        self.auth = auth
+        self.tls = tls
+
+    def publish_single(self, topic, msg):
+        publish.single(topic, msg, hostname=self.hostname,
+                       port=self.port, client_id=self.client_id,
+                       keepalive=self.keepalive, will=self.will,
+                       auth=self.auth, tls=self.tls)
+
+    def publish_multiple(self, topic, msg):
+        publish.multiple(topic, msg, hostname=self.hostname,
+                         port=self.port, client_id=self.client_id,
+                         keepalive=self.keepalive, will=self.will,
+                         auth=self.auth, tls=self.tls)
+
+
+def process_event(event, base_topic):
+    pieces = [base_topic]
+    if 'project' in event:
+        pieces.append(event['project'])
+        if 'event_type' in event:
+            pieces.append(event['event-type'])
+            pieces.append(event['bug-number'])
+    topic = "/".join(pieces)
+    msg = json.dumps(event)
+    return msg, topic
+
+
+def main():
+    config = configparser.ConfigParser()
+    config.read(sys.argv[1])
+
+    # Configure MQTT Connection
+    if config.has_option('mqtt', 'port'):
+        mqtt_port = config.get('mqtt', 'port')
+    else:
+        mqtt_port = 1883
+    if config.has_option('mqtt', 'keepalive'):
+        keepalive = config.get('mqtt', 'keepalive')
+    else:
+        keepalive = 60
+    # Configure MQTT auth
+    auth = None
+    if config.has_option('mqtt', 'username'):
+        mqtt_username = config.get('mqtt', 'username')
+    else:
+        mqtt_username = None
+    if config.has_option('mqtt', 'password'):
+        mqtt_password = config.get('mqtt', 'password')
+    else:
+        mqtt_password = None
+    if mqtt_username:
+        auth = {'username': mqtt_username}
+        if mqtt_password:
+            auth['password'] = mqtt_password
+    base_topic = config.get('mqtt', 'topic')
+
+    mqttqueue = PushMQTT(
+        config.get('mqtt', 'hostname'),
+        port=mqtt_port,
+        keepalive=keepalive,
+        auth=auth)
+
+    launchpad = lp.LPImap('', '')
+    while True:
+        events = launchpad.getEvents()
+        for event in events:
+            msg, topic = process_event(event, base_topic)
+            mqttqueue.publish_single(topic, msg)
+        launchpad.imap.idle()
+
+if __name__ == "__main__":
+    main()

lpmqtt/lp.py

@@ -0,0 +1,81 @@
+# Copyright 2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import email
+import imaplib2 as imaplib
+
+
+class LPImapWatcher(object):
+
+    def __init__(self, imap_user, imap_pass, delete=False):
+        super(LPImapWatcher, self).__init__()
+        self.imap = imaplib.IMAP4()
+        self.imap.login(imap_user, imap_pass)
+        self.delete = delete
+
+    def getEvents(self):
+        self.imap.select()
+        events = self._find_new_email()
+        return events
+
+    def _process_bug(self, message):
+        event = {}
+        event['event-type'] = 'bug'
+        event['commenters'] = message['X-Launchpad-Bug-Commenters'].split(' ')
+        event['bug-reporter'] = message['X-Launchpad-Bug-Reporter']
+        event['bug-modifier'] = message['X-Launchpad-Bug-Modifier']
+        event['tags'] = message['X-Launchpad-Bug-Tags'].split(' ')
+        bug_info = message['X-Launchpad-Bug'].split(';')
+        for info in bug_info:
+            clean_info = info.lstrip()
+            key_value = clean_info.split('=')
+            key = key_value[0]
+            value = key_value[1]
+            if key == 'product':
+                event['project'] = value
+            else:
+                event[key] = value
+        return event
+
+    def _process_msg(self, data):
+        event = {}
+        message = email.message_from_string(data[1])
+        if 'X-Generated-By' not in message or 'Launchpad' not in message[
+            'X-Generated-By']:
+            return event
+        # Mark the message as read
+        email_id = data[0].split()[0]
+        self.imap.fetch(email_id, '(RFC822)')
+        event_type = message['X-Launchpad-Notification-Type']
+        if event_type == 'bug':
+            event = self._proccess_bug(message)
+        else:
+            for header in message:
+                if header.startswith('X-Launchpad'):
+                    event[header] = message[header]
+                event['body'] = message.get_payload()
+        return event
+
+    def _find_new_email(self):
+        events = []
+        typ, msg_ids = self.imap.search(None, "UNSEEN")
+        for msg in msg_ids[0].split():
+            typ, data = self.imap.fetch(msg, "(BODY.PEEK[HEADER])")
+            output = self._process_msg(data, msg)
+            if self.delete:
+                self.imap.store(msg, '+FLAGS', '\\Deleted')
+            events.append(output)
+        if self.delete:
+            self.imap.expunge()
+        return events

requirements.txt

@@ -0,0 +1,8 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+pbr>=1.6 # Apache-2.0
+six>=1.9.0 # MIT
+paho-mqtt>=1.1
+gerritlib>=0.6.0
+imaplib2>=2.45.0

setup.cfg

@@ -0,0 +1,37 @@
+[metadata]
+name = lpmqtt
+summary = Publish Launchpad Event Stream on MQTT
+description-file =
+    README.rst
+author = Matthew Treinish
+author-email = mtreinish@kortar.org
+classifier =
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    Intended Audience :: Developers
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.4
+
+[files]
+packages =
+    lpmqtt
+
+[entry_points]
+console_scripts =
+    lpmqtt = lpmqtt.daemon:main
+
+[build_sphinx]
+all_files = 1
+build-dir = doc/build
+source-dir = doc/source
+
+[pbr]
+warnerrors = True
+
+[wheel]
+universal = 1

setup.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=['pbr>=1.8'],
+    pbr=True)

test-requirements.txt

@@ -0,0 +1,10 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+hacking<0.11,>=0.10.2 # Apache-2.0
+
+coverage>=3.6 # Apache-2.0
+sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
+oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
+os-testr>=0.6.0

tox.ini

@@ -0,0 +1,36 @@
+[tox]
+minversion = 1.6
+envlist = py33,py34,py26,py27,pypy,pep8
+skipsdist = True
+
+[testenv]
+usedevelop = True
+install_command = pip install -U {opts} {packages}
+setenv =
+   VIRTUAL_ENV={envdir}
+whitelist_externals = find
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+         find . -type f -name "*.pyc" -delete
+         ostestr {posargs}
+
+[testenv:pep8]
+commands = flake8
+
+[testenv:venv]
+commands = {posargs}
+
+[testenv:cover]
+commands = python setup.py testr --coverage --testr-args='{posargs}'
+
+[testenv:docs]
+commands = python setup.py build_sphinx
+
+[flake8]
+# E123, E125 skipped as they are invalid PEP-8.
+
+show-source = True
+ignore = E123,E125,E129
+builtins = _
+exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build