Stackalytics deployment clean up

* Scripts and conf locations are set in setup.cfg
* Flask-styled is replaced by oslo config
* Scripts from bin are replaced by one generated by entry_points

Change-Id: I26620b8d945d331b2d5abebce41ca177f3e93571

bin/dashboard

@@ -1,11 +0,0 @@
-#!.venv/bin/python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-import os
-import sys
-sys.path.insert(0, os.getcwd())
-
-from dashboard.web import app
-
-
-app.run()

bin/processor

@@ -1,11 +0,0 @@
-#!.venv/bin/python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-import os
-import sys
-sys.path.insert(0, os.getcwd())
-
-from stackalytics.processor.main import main
-
-
-main()

dashboard/web.py

@@ -23,9 +23,12 @@ import urllib
 
 import flask
 from flask.ext import gravatar as gravatar_ext
+from oslo.config import cfg
 import time
 
 from dashboard import memory_storage
+from stackalytics.openstack.common import log as logging
+from stackalytics.processor import config
 from stackalytics.processor import persistent_storage
 from stackalytics.processor import runtime_storage
 from stackalytics.processor import user_utils
@@ -33,10 +36,6 @@ from stackalytics.processor import user_utils
 
 # Constants and Parameters ---------
 
-DEBUG = True
-RUNTIME_STORAGE_URI = 'memcached://127.0.0.1:11211'
-PERSISTENT_STORAGE_URI = 'mongodb://localhost'
-
 DEFAULTS = {
     'metric': 'commits',
     'release': 'havana',
@@ -62,16 +61,33 @@ app = flask.Flask(__name__)
 app.config.from_object(__name__)
 app.config.from_envvar('DASHBOARD_CONF', silent=True)
 
+LOG = logging.getLogger(__name__)
+
+conf = cfg.CONF
+conf.register_opts(config.OPTS)
+logging.setup('dashboard')
+LOG.info('Logging enabled')
+
+conf_file = os.getenv('STACKALYTICS_CONF')
+if (conf_file is None) or (not os.path.isfile(conf_file)):
+    conf_file = '/etc/stackalytics/stackalytics.conf'
+
+if os.path.isfile(conf_file):
+    conf(default_config_files=[conf_file])
+    app.config['DEBUG'] = cfg.CONF.debug
+else:
+    LOG.warn('Conf file is empty or not exist')
+
 
 def get_vault():
     vault = getattr(app, 'stackalytics_vault', None)
     if not vault:
         vault = {}
         vault['runtime_storage'] = runtime_storage.get_runtime_storage(
-            RUNTIME_STORAGE_URI)
+            cfg.CONF.runtime_storage_uri)
         vault['persistent_storage'] = (
             persistent_storage.get_persistent_storage(
-                PERSISTENT_STORAGE_URI))
+                cfg.CONF.persistent_storage_uri))
         vault['memory_storage'] = memory_storage.get_memory_storage(
             memory_storage.MEMORY_STORAGE_CACHED,
             vault['runtime_storage'].get_update(os.getpid()))
@@ -208,7 +224,7 @@ def exception_handler():
             try:
                 return f(*args, **kwargs)
             except Exception as e:
-                print e
+                LOG.debug(e)
                 flask.abort(404)
 
         return decorated_function
@@ -510,5 +526,9 @@ def make_commit_message(record):
 gravatar = gravatar_ext.Gravatar(app, size=100, rating='g',
                                  default='wavatar')
 
+
+def main():
+    app.run(cfg.CONF.listen_host, cfg.CONF.listen_port)
+
 if __name__ == '__main__':
-    app.run('0.0.0.0')
+    main()

etc/dashboard.conf

@@ -1,5 +0,0 @@
-#
-# Configuration of stackalytics dashboard
-#
-
-DEBUG = True

etc/stackalytics.conf

@@ -3,10 +3,10 @@
 # debug = False
 
 # Default data
-# default-data = etc/default_data.json
+# default-data = /etc/stackalytics/default_data.json
 
 # The folder that holds all project sources to analyze
-# sources_root = ../metric-root-tmp
+# sources_root = /var/run/stackalytics
 
 # Runtime storage URI
 # runtime_storage_uri = memcached://127.0.0.1:11211
@@ -21,4 +21,10 @@
 # repo_poll_period = 300
 
 # Address of update handler
-# frontend_update_address = http://user:user@localhost/update/%s
+# frontend_update_address = http://user:user@localhost/update/%s
+
+# Hostname where dashboard listens on
+# listen_host = 127.0.0.1
+
+# Port where dashboard listens on
+# listen_port = 8080

setup.cfg

@@ -4,7 +4,7 @@ version = 0.1
 summary = OpenStack analytics dashboard
 description-file =
     README.rst
-author = OpenStack Stackalytics Project
+author = OpenStack
 author-email = openstack-dev@lists.openstack.org
 home-page = http://www.openstack.org/
 classifier =
@@ -21,6 +21,10 @@ classifier =
 packages =
     dashboard
     stackalytics
+data_files =
+    etc/stackalytics =
+        etc/stackalytics.conf
+        etc/default_data.json
 
 [global]
 setup-hooks =
@@ -28,4 +32,5 @@ setup-hooks =
 
 [entry_points]
 console_scripts =
-    stackalytics-dashboard = dashboard.dashboard:main
+    stackalytics-dashboard = dashboard.web:main
+    stackalytics-processor = stackalytics.processor.main:main

stackalytics/processor/commit_processor.py

@@ -79,11 +79,11 @@ class CachedProcessor(CommitProcessor):
     def _unknown_user_email(self, email):
 
         lp_profile = None
-        if not re.match(r'[^@]+@[^@]+\.[^@]+', email):
+        if not re.match(r'[\w\d_\.-]+@([\w\d_\.-]+\.)+[\w]+', email):
             LOG.debug('User email is not valid %s' % email)
         else:
             LOG.debug('Lookup user email %s at Launchpad' % email)
-            lp = launchpad.Launchpad.login_anonymously(cfg.CONF.launchpad_user)
+            lp = launchpad.Launchpad.login_anonymously('stackalytics')
             try:
                 lp_profile = lp.people.getByEmail(email=email)
             except Exception as error:

stackalytics/processor/config.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo.config import cfg
+
+OPTS = [
+    cfg.StrOpt('default-data', default='etc/default_data.json',
+               help='Default data'),
+    cfg.StrOpt('sources-root', default='/var/run/stackalytics',
+               help='The folder that holds all project sources to analyze'),
+    cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
+               help='Storage URI'),
+    cfg.StrOpt('frontend-update-address',
+               default='http://user:user@localhost/update/%s',
+               help='Address of update handler'),
+    cfg.StrOpt('repo-poll-period', default='300',
+               help='Repo poll period in seconds'),
+    cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost',
+               help='URI of persistent storage'),
+    cfg.BoolOpt('sync-default-data', default=False,
+                help='Update persistent storage with default data. '
+                     'Existing data is not overwritten'),
+    cfg.BoolOpt('force-sync-default-data', default=False,
+                help='Completely overwrite persistent storage with the '
+                     'default data'),
+    cfg.BoolOpt('filter-robots', default=True,
+                help='Filter out commits from robots'),
+    cfg.StrOpt('listen-host', default='127.0.0.1',
+               help='The address dashboard listens on'),
+    cfg.IntOpt('listen-port', default=8080,
+               help='The port dashboard listens on'),
+]

stackalytics/processor/main.py

@@ -19,6 +19,7 @@ from psutil import _error
 
 from stackalytics.openstack.common import log as logging
 from stackalytics.processor import commit_processor
+from stackalytics.processor import config
 from stackalytics.processor import persistent_storage
 from stackalytics.processor import runtime_storage
 from stackalytics.processor import vcs
@@ -26,27 +27,6 @@ from stackalytics.processor import vcs
 
 LOG = logging.getLogger(__name__)
 
-OPTS = [
-    cfg.StrOpt('default-data', default='etc/default_data.json',
-               help='Default data'),
-    cfg.StrOpt('sources-root', default=None, required=True,
-               help='The folder that holds all project sources to analyze'),
-    cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
-               help='Storage URI'),
-    cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost',
-               help='URI of persistent storage'),
-    cfg.BoolOpt('sync-default-data', default=False,
-                help='Update persistent storage with default data. '
-                     'Existing data is not overwritten'),
-    cfg.BoolOpt('force-sync-default-data', default=False,
-                help='Completely overwrite persistent storage with the '
-                     'default data'),
-    cfg.StrOpt('launchpad-user', default='stackalytics-bot',
-               help='User to access Launchpad'),
-    cfg.BoolOpt('filter-robots', default=True,
-                help='Filter out commits from robots'),
-]
-
 
 def get_pids():
     uwsgi_dict = {}
@@ -107,8 +87,8 @@ def update_repos(runtime_storage, persistent_storage):
 def main():
     # init conf and logging
     conf = cfg.CONF
-    conf.register_cli_opts(OPTS)
+    conf.register_cli_opts(config.OPTS)
-    conf.register_opts(OPTS)
+    conf.register_opts(config.OPTS)
     conf()
 
     logging.setup('stackalytics')