From 8c4634366d6342145a4f9a574760dcb4b7023d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Vachon?= Date: Wed, 8 Apr 2015 11:54:05 -0400 Subject: [PATCH] Auto-reload and pipeline Change-Id: Ib5e062fdedefda2352d81daa5af2a6f2fca44ca0 --- Dockerfile | 1 + etc/surveil/api_paste.ini | 15 +++ requirements.txt | 4 + surveil/api/app.py | 41 +++++- surveil/cmd/api.py | 127 +++++++++++++++++- .../etc/supervisor/conf.d/supervisor.conf | 2 +- 6 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 etc/surveil/api_paste.ini diff --git a/Dockerfile b/Dockerfile index 742bd47..81cdc9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ ADD setup.py /surveil/setup.py ADD setup.cfg /surveil/setup.cfg ADD README.rst /surveil/README.rst ADD .git /surveil/.git +ADD etc/surveil /etc/surveil # Install RUN pip install -r /surveil/requirements.txt diff --git a/etc/surveil/api_paste.ini b/etc/surveil/api_paste.ini new file mode 100644 index 0000000..1fac210 --- /dev/null +++ b/etc/surveil/api_paste.ini @@ -0,0 +1,15 @@ +# Surveil API WSGI Pipeline +# Define the filters that make up the pipeline for processing WSGI requests + +# Remove authtoken from the pipeline if you don't want to use keystone authentication +[pipeline:main] +pipeline = api-server + +[app:api-server] +paste.app_factory = surveil.api.app:app_factory + +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory + +[filter:request_id] +paste.filter_factory = oslo.middleware:RequestId.factory diff --git a/requirements.txt b/requirements.txt index fdd7db3..f66cd0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,7 @@ pymongo>=2.7.2 wsme requests watchdog +oslo.config +oslo.middleware +keystonemiddleware +PasteDeploy diff --git a/surveil/api/app.py b/surveil/api/app.py index d1dfbde..7e14396 100644 --- a/surveil/api/app.py +++ b/surveil/api/app.py @@ -12,17 +12,52 @@ # License for the specific language governing permissions and limitations # under the License. +import os + +from paste import deploy import pecan -def setup_app(config): +def get_config_filename(): + abspath = os.path.abspath(__file__) + path = os.path.dirname(abspath) + filename = "config.py" + return os.path.join(path, filename) - app_conf = dict(config.app) + +def get_pecan_config(): + # Set up the pecan configuration + return pecan.configuration.conf_from_file(get_config_filename()) + + +def setup_app(pecan_config): + app_conf = dict(pecan_config.app) app = pecan.make_app( app_conf.pop('root'), - logging=getattr(config, 'logging', {}), + logging=getattr(pecan_config, 'logging', {}), **app_conf ) return app + + +def load_app(): + return deploy.loadapp('config:/etc/surveil/api_paste.ini') + + +def app_factory(global_config, **local_conf): + return VersionSelectorApplication() + + +class VersionSelectorApplication(object): + def __init__(self): + pc = get_pecan_config() + + self.v1 = setup_app(pecan_config=pc) + self.v2 = setup_app(pecan_config=pc) + + def __call__(self, environ, start_response): + if environ['PATH_INFO'].startswith('/v1/'): + return self.v1(environ, start_response) + return self.v2(environ, start_response) diff --git a/surveil/cmd/api.py b/surveil/cmd/api.py index a99a671..ad62e45 100644 --- a/surveil/cmd/api.py +++ b/surveil/cmd/api.py @@ -17,11 +17,130 @@ import os import subprocess import sys +import threading +import time +from wsgiref import simple_server -from surveil import api +from oslo_config import cfg + +import surveil.api.app as app + +CONF = cfg.CONF + +OPTS = [ + cfg.StrOpt( + 'api_paste_config', + default="api_paste.ini", + help="Configuration file for WSGI definition of API." + ), +] + +CONF.register_opts(OPTS) + + +class ServerManager: + + def __init__(self): + self.config = {} + self.config_file = "" + self.server_process = None + self.should_run = True + + def run(self, pecan_config, config_file): + self.config = pecan_config + self.config_file = config_file + + if '--reload' in sys.argv: + self.watch_and_spawn() + else: + self.start_server() + + def create_subprocess(self): + self.server_process = subprocess.Popen(['surveil-api']) + + def start_server(self): + pecan_app = app.load_app() + host, port = self.config.server.host, self.config.server.port + srv = simple_server.make_server(host, port, pecan_app) + srv.serve_forever() + + def watch_and_spawn(self): + import watchdog.events as events + import watchdog.observers as observers + + print('Monitoring for changes...') + self.create_subprocess() + + parent = self + + class AggressiveEventHandler(events.FileSystemEventHandler): + + def __init__(self): + self.wait = False + + def should_reload(self, event): + for t in ( + events.FileSystemMovedEvent, + events.FileModifiedEvent, + events.DirModifiedEvent + ): + if isinstance(event, t): + return True + return False + + def ignore_events_one_sec(self): + if not self.wait: + self.wait = True + t = threading.Thread(target=self.wait_one_sec) + t.start() + + def wait_one_sec(self): + time.sleep(1) + self.wait = False + + def on_modified(self, event): + if self.should_reload(event) and not self.wait: + print("Some source files have been modified") + print("Restarting server...") + self.ignore_events_one_sec() + parent.server_process.kill() + parent.create_subprocess() + + # Determine a list of file paths to monitor + paths = self.paths_to_monitor() + + event_handler = AggressiveEventHandler() + for path, recurse in paths: + observer = observers.Observer() + observer.schedule( + event_handler, + path=path, + recursive=recurse + ) + observer.start() + + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass + + def paths_to_monitor(self): + paths = [] + + for package_name in getattr(self.config.app, 'modules', []): + module = __import__(package_name, fromlist=['app']) + if hasattr(module, 'app') and hasattr(module.app, 'setup_app'): + paths.append(( + os.path.dirname(module.__file__), + True + )) + break + + paths.append((os.path.dirname(self.config.__file__), False)) + return paths def main(): - filename = os.path.join(os.path.dirname(api.__file__), "config.py") - subprocess.Popen(['pecan', 'serve', '--reload', filename], - stdin=sys.stdout, stdout=sys.stdout) + srv = ServerManager() + srv.run(app.get_pecan_config(), app.get_config_filename()) diff --git a/tools/docker/surveil_container/etc/supervisor/conf.d/supervisor.conf b/tools/docker/surveil_container/etc/supervisor/conf.d/supervisor.conf index 4304615..c28a1b8 100644 --- a/tools/docker/surveil_container/etc/supervisor/conf.d/supervisor.conf +++ b/tools/docker/surveil_container/etc/supervisor/conf.d/supervisor.conf @@ -2,7 +2,7 @@ nodaemon=true [program:surveil] -command=/bin/sh -c "surveil-api" +command=/bin/sh -c "surveil-api --reload" [program:surveil-init] command=/bin/sh -c "sleep 10 && surveil-init"