diff --git a/etc/libra_api.py b/etc/libra_api.py new file mode 100644 index 00000000..09be9e8a --- /dev/null +++ b/etc/libra_api.py @@ -0,0 +1,71 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Server Specific Configurations +server = { + 'port': '8080', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'libra.api.controllers.root.RootController', + 'modules': ['libra.api'], + 'static_root': '%(confdir)s/public', + 'template_path': '%(confdir)s/api/templates', + 'debug': True, + 'errors': { + 404: '/error/404', + '__force_dict__': True + } +} + +logging = { + 'loggers': { + 'root': {'level': 'INFO', 'handlers': ['console']}, + 'api': {'level': 'DEBUG', 'handlers': ['console']} + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + } + }, + 'formatters': { + 'simple': { + 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' + '[%(threadName)s] %(message)s') + } + } +} + +database = { + 'username': 'root', + 'password': 'testaburger', + 'host': 'localhost', + 'schema': 'lbaas' +} + +gearman = { + 'server': ['localhost:4730'], +} + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/libra/api/app.py b/libra/api/app.py new file mode 100644 index 00000000..eb36d498 --- /dev/null +++ b/libra/api/app.py @@ -0,0 +1,34 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pecan import make_app +from libra.api import model + + +def setup_app(config): + + model.init_model() + + return make_app( + config.app.root, + static_root=config.app.static_root, + template_path=config.app.template_path, + logging=getattr(config, 'logging', {}), + debug=getattr(config.app, 'debug', False), + force_canonical=getattr(config.app, 'force_canonical', True), + guess_content_type_from_ext=getattr( + config.app, + 'guess_content_type_from_ext', + True), + ) diff --git a/libra/api/controllers/__init__.py b/libra/api/controllers/__init__.py new file mode 100644 index 00000000..92bd912f --- /dev/null +++ b/libra/api/controllers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/libra/api/controllers/connection_throttle.py b/libra/api/controllers/connection_throttle.py new file mode 100644 index 00000000..868505f2 --- /dev/null +++ b/libra/api/controllers/connection_throttle.py @@ -0,0 +1,66 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pecan import expose, response +from pecan.rest import RestController + +from libra.api.model.responses import Responses + + +class ConnectionThrottleController(RestController): + """functions for /loadbalancers/{loadBalancerId}/connectionthrottle/* + routing""" + + @expose('json') + def get(self, load_balancer_id): + """List connection throttling configuration. + + :param load_balancer_id: id of lb + + Url: + GET /loadbalancers/{load_balancer_id}/connectionthrottle + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.ConnectionThrottle.get + + @expose('json') + def post(self, load_balancer_id, *args): + """Update throttling configuration. + + :param load_balancer_id: id of lb + :param *args: holds the posted json or xml + + Url: + PUT /loadbalancers/loadBalancerId/connectionthrottle + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.ConnectionThrottle.get + + @expose() + def delete(self, loadbalancer_id): + """Remove connection throttling configurations. + + :param load_balancer_id: id of lb + + Url: + DELETE /loadbalancers/loadBalancerId/connectionthrottle + + Returns: void + """ + response.status = 201 diff --git a/libra/api/controllers/health_monitor.py b/libra/api/controllers/health_monitor.py new file mode 100644 index 00000000..8affea49 --- /dev/null +++ b/libra/api/controllers/health_monitor.py @@ -0,0 +1,65 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pecan import expose, response +from pecan.rest import RestController + +from libra.api.model.responses import Responses + + +class HealthMonitorController(RestController): + """functions for /loadbalancers/{loadBalancerId}/healthmonitor/* routing""" + + @expose('json') + def get(self, load_balancer_id): + """Retrieve the health monitor configuration, if one exists. + + :param load_balancer_id: id of lb + + Url: + GET /loadbalancers/{load_balancer_id}/healthmonitor + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.get + + @expose('json') + def post(self, load_balancer_id, *args): + """Update the settings for a health monitor. + + :param load_balancer_id: id of lb + :param *args: holds the posted json or xml data + + Url: + PUT /loadbalancers/{load_balancer_id}/healthmonitor + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.get + + @expose() + def delete(self, load_balancer_id): + """Remove the health monitor. + + :param load_balancer_id: id of lb + + Url: + DELETE /loadbalancers/{load_balancer_id}/healthmonitor + + Returns: void + """ + response.status = 201 diff --git a/libra/api/controllers/load_balancers.py b/libra/api/controllers/load_balancers.py new file mode 100644 index 00000000..9311b1bc --- /dev/null +++ b/libra/api/controllers/load_balancers.py @@ -0,0 +1,313 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +#import gearman.errors + +import json +#import socket +#import time +# pecan imports +from pecan import expose, abort, response +from pecan.rest import RestController +# other controllers +from nodes import NodesController +from health_monitor import HealthMonitorController +from session_persistence import SessionPersistenceController +from connection_throttle import ConnectionThrottleController +#from sqlalchemy.orm import aliased +# default response objects +from libra.api.model.responses import Responses +# models +from libra.api.model.lbaas import LoadBalancer, Device, Node, session +from libra.api.library.gearman_client import gearman_client + + +class LoadBalancersController(RestController): + """functions for /loadbalancer routing""" + loadbalancer_status = ( + 'ACTIVE', + 'BUILD', + 'PENDING_UPDATE', + 'PENDING_DELETE', + 'DELETED', + 'SUSPENDED', + 'ERROR' + ) + + """nodes subclass linking + controller class for urls that look like + /loadbalancers/{loadBalancerId}/nodes/* + """ + nodes = NodesController() + + """healthmonitor instance + controller class for urls that start with + /loadbalancers/{loadBalancerId}/healthmonitor/* + """ + healthmonitor = HealthMonitorController() + + """healthmonitor instance + controller class for urls that start with + /loadbalancers/{loadBalancerId}/sessionpersistence/* + """ + sessionpersistence = SessionPersistenceController() + + """connectionthrottle instance + controller class for urls that start with + /loadbalancers/{loadBalancerId}/connectionthrottle/* + """ + connectionthrottle = ConnectionThrottleController() + + @expose('json') + def get(self, load_balancer_id=None): + """Fetches a list of load balancers or the details of one balancer if + load_balancer_id is not empty. + + :param load_balancer_id: id of lb we want to get, if none it returns a + list of all + + Url: + GET /loadbalancers + List all load balancers configured for the account. + + Url: + GET /loadbalancers/{load_balancer_id} + List details of the specified load balancer. + + Returns: dict + """ + # have not implimented the keystone middleware so we don't know the + # tenantid + tenant_id = 80074562416143 + + # if we don't have an id then we want a list of them own by this tenent + if not load_balancer_id: + #return Responses.LoadBalancers.get + load_balancers = {'loadBalancers': session.query( + LoadBalancer.name, LoadBalancer.id, LoadBalancer.protocol, + LoadBalancer.port, LoadBalancer.algorithm, + LoadBalancer.status, LoadBalancer.created, + LoadBalancer.updated + ).filter_by(tenantid=tenant_id).all()} + else: + #return Responses.LoadBalancers.detail + load_balancers = session.query( + LoadBalancer.name, LoadBalancer.id, LoadBalancer.protocol, + LoadBalancer.port, LoadBalancer.algorithm, + LoadBalancer.status, LoadBalancer.created, + LoadBalancer.updated + ).join(LoadBalancer.devices).\ + filter(LoadBalancer.tenantid == tenant_id).\ + filter(LoadBalancer.id == load_balancer_id).\ + first()._asdict() + + virtualIps = session.query( + Device.id, Device.floatingIpAddr + ).join(LoadBalancer.devices).\ + filter(LoadBalancer.tenantid == tenant_id).\ + filter(LoadBalancer.id == load_balancer_id).\ + all() + + load_balancers['virtualIps'] = [] + for item in virtualIps: + vip = item._asdict() + vip['type'] = 'PUBLIC' + vip['ipVersion'] = 'IPV4' + load_balancers['virtualIps'].append(vip) + + nodes = session.query( + Node.id, Node.address, Node.port, Node.status, Node.enabled + ).join(LoadBalancer.nodes).\ + filter(LoadBalancer.tenantid == tenant_id).\ + filter(LoadBalancer.id == load_balancer_id).\ + all() + + load_balancers['nodes'] = [] + for item in nodes: + node = item._asdict() + if node['enabled'] == 1: + node['condition'] = 'ENABLED' + else: + node['condition'] = 'DISABLED' + del node['enabled'] + load_balancers['nodes'].append(node) + + if load_balancers is None: + return Responses.not_found + else: + return load_balancers + + @expose('json') + def post(self, load_balancer_id=None, **kwargs): + """Accepts edit if load_balancer_id isn't blank or create load balancer + posts. + + :param load_balancer_id: id of lb + :param *args: holds the posted json or xml data + + Urls: + POST /loadbalancers/{load_balancer_id} + PUT /loadbalancers + + Notes: + curl -i -H "Accept: application/json" -X POST \ + -d "data={"name": "my_lb"}" \ + http://dev.server:8080/loadbalancers/100 + + Returns: dict + """ + # have not implimented the keystone middleware so we don't know the + # tenantid + tenant_id = 80074562416143 + + # load input + data = json.loads(kwargs['data']) + # TODO validate input data + + # if we don't have an id then we want to create a new lb + if not load_balancer_id: + lb = LoadBalancer() + + # find free device + device = Device.find_free_device() + + if device is None: + response.status = 503 + return Responses.service_unavailable + + lb.device = device.id + lb.tenantid = tenant_id + + lb.update_from_json(data) + + # write to database + session.add(lb) + session.flush() + #refresh the lb record so we get the id back + session.refresh(lb) + + # now save the loadbalancer_id to the device and switch its status + # to online + device.loadbalancers = lb.id + device.status = "ONLINE" + + else: + # grab the lb + lb = session.query(LoadBalancer)\ + .filter_by(id=load_balancer_id).first() + + if lb is None: + response.status = 400 + return Responses.not_found + + lb.update_from_json(data) + + try: + session.flush() + + # trigger gearman client to create new lb + result = gearman_client.submit_job('UPDATE', lb.output_to_json()) + + # do something with result + if result: + pass + + response.status = 200 + return self.get() + except: + response.status = 503 + return Responses.service_unavailable + + @expose() + def delete(self, load_balancer_id): + """Remove a load balancer from the account. + + :param load_balancer_id: id of lb + + Urls: + DELETE /loadbalancers/{load_balancer_id} + + Notes: + curl -i -H "Accept: application/json" -X DELETE + http://dev.server:8080/loadbalancers/1 + + Returns: None + """ + # grab the lb + lb = session.query(LoadBalancer)\ + .filter_by(id=load_balancer_id).first() + + if lb is None: + response.status = 400 + return Responses.not_found + + try: + session.flush() + + # trigger gearman client to create new lb + result = gearman_client.submit_job('DELETE', lb.output_to_json()) + + if result: + pass + + response.status = 200 + + session.delete(lb) + session.commit() + + return self.get() + except: + response.status = 503 + return Responses.service_unavailable + + def virtualips(self, load_balancer_id): + """Returns a list of virtual ips attached to a specific Load Balancer. + + :param load_balancer_id: id of lb + + Url: + GET /loadbalancers/{load_balancer_id}/virtualips + + Returns: dict + """ + return Responses.LoadBalancers.virtualips + + def usage(self, load_balancer_id): + """List current and historical usage. + + :param load_balancer_id: id of lb + + Url: + GET /loadbalancers/{load_balancer_id}/usage + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.usage + + @expose('json') + def _lookup(self, primary_key, *remainder): + """Routes more complex url mapping. + + :param primary_key: value to look up or pass + :param *remainder: remaining args + + Raises: 404 + """ + #student = get_student_by_primary_key(primary_key) + #if student: + # return StudentController(student), remainder + #else: + abort(404) diff --git a/libra/api/controllers/nodes.py b/libra/api/controllers/nodes.py new file mode 100644 index 00000000..b8798f15 --- /dev/null +++ b/libra/api/controllers/nodes.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pecan import expose, response +from pecan.rest import RestController +#default response objects +#from libra.api.model.lbaas import Device, LoadBalancer, Node, session +from libra.api.model.responses import Responses + + +class NodesController(RestController): + """Functions for /loadbalancers/{load_balancer_id}/nodes/* routing""" + + @expose('json') + def get(self, load_balancer_id, node_id=None): + """List node(s) configured for the load balancer OR if + node_id == None .. Retrieve the configuration of node {node_id} of + loadbalancer {load_balancer_id}. + :param load_balancer_id: id of lb + :param node_id: id of node (optional) + + Urls: + GET /loadbalancers/{load_balancer_id}/nodes + GET /loadbalancers/{load_balancer_id}/nodes/{node_id} + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.Nodes.get + + @expose('json') + def post(self, load_balancer_id, node_id=None, *args): + """Adds a new node to the load balancer OR Modify the configuration + of a node on the load balancer. + + :param load_balancer_id: id of lb + :param node_id: id of node (optional) when missing a new node is added. + :param *args: holds the posted json or xml data + + Urls: + POST /loadbalancers/{load_balancer_id}/nodes + PUT /loadbalancers/{load_balancer_id}/nodes/{node_id} + + Returns: dict of the full list of nodes or the details of the single + node + """ + response.status = 201 + return Responses.LoadBalancers.Nodes.get + + @expose() + def delete(self, load_balancer_id, node_id): + """Remove a node from the load balancer. + + :param load_balancer_id: id of lb + :param node_id: id of node + + Url: + DELETE /loadbalancers/{load_balancer_id}/nodes/{node_id} + + Returns: None + """ + response.status = 201 diff --git a/libra/api/controllers/root.py b/libra/api/controllers/root.py new file mode 100644 index 00000000..44c02d83 --- /dev/null +++ b/libra/api/controllers/root.py @@ -0,0 +1,56 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pecan import expose, response +from load_balancers import LoadBalancersController +from libra.api.model.responses import Responses + + +class RootController(object): + """root control object.""" + + @expose('json') + def _default(self): + """default route.. acts as catch all for any wrong urls. + For now it returns a 404 because no action is defined for /""" + response.status = 201 + return Responses._default + + @expose('json') + def protocols(self): + """Lists all supported load balancing protocols. + + Url: + GET /protocols + + Returns: dict + """ + response.status = 201 + return Responses.protocols + + @expose('json') + def algorithms(self): + """List all supported load balancing algorithms. + + Url: + GET /algorithms + + Returns: dict + """ + response.status = 201 + return Responses.algorithms + + #pecan uses this controller class for urls that start with /loadbalancers + loadbalancers = LoadBalancersController() diff --git a/libra/api/controllers/session_persistence.py b/libra/api/controllers/session_persistence.py new file mode 100644 index 00000000..efe79d01 --- /dev/null +++ b/libra/api/controllers/session_persistence.py @@ -0,0 +1,65 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pecan import expose, response +from pecan.rest import RestController +from libra.api.model.responses import Responses + + +class SessionPersistenceController(RestController): + """SessionPersistenceController + functions for /loadbalancers/{loadBalancerId}/sessionpersistence/* routing + """ + + @expose('json') + def get(self, load_balancer_id): + """List session persistence configuration.get + + :param load_balancer_id: id of lb + + Url: + GET /loadbalancers/{load_balancer_id}/sessionpersistence + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.SessionPersistence.get + + @expose('json') + def post(self, load_balancer_id): + """Enable session persistence. + + :param load_balancer_id: id of lb + + Url: + PUT /loadbalancers/{load_balancer_id}/sessionpersistence + + Returns: dict + """ + response.status = 201 + return Responses.LoadBalancers.SessionPersistence.get + + @expose('json') + def delete(self, load_balancer_id): + """Disable session persistence. + + :param load_balancer_id: id of lb + + Url: + DELETE /loadbalancers/{load_balancer_id}/sessionpersistence + + Returns: dict + """ + response.status = 201 diff --git a/libra/api/library/__init__.py b/libra/api/library/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libra/api/library/gearman_client.py b/libra/api/library/gearman_client.py new file mode 100644 index 00000000..2e6e022b --- /dev/null +++ b/libra/api/library/gearman_client.py @@ -0,0 +1,29 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from libra.common.json_gearman import JSONGearmanClient +from pecan import conf + + +gearman_client = JSONGearmanClient(conf.gearman.server) + +gearman_workers = [ + 'UPDATE', # Create/Update a Load Balancer. + 'SUSPEND', # Suspend a Load Balancer. + 'ENABLE', # Enable a suspended Load Balancer. + 'DELETE', # Delete a Load Balancer. + 'DISCOVER', # Return service discovery information. + 'ARCHIVE', # Archive LB log files. + 'STATS' # Get load balancer statistics. +] diff --git a/libra/api/main.py b/libra/api/main.py deleted file mode 100644 index a39e9fc2..00000000 --- a/libra/api/main.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import daemon -import daemon.pidfile -import daemon.runner -import grp -import lockfile -import os -import pwd - -from libra.common.options import Options, setup_logging - - -class APIServer(object): - def __init__(self, logger, args): - self.logger = logger - self.args = args - - -def main(): - options = Options('api', 'API Daemon') - args = options.run() - - logger = setup_logging('api_server', args) - server = APIServer(logger, args) - - if args.nodaemon: - server.main() - else: - pidfile = daemon.pidfile.TimeoutPIDLockFile(args.pid, 10) - if daemon.runner.is_pidfile_stale(pidfile): - logger.warning("Cleaning up stale PID file") - pidfile.break_lock() - context = daemon.DaemonContext( - working_directory='/etc/libra', - umask=0o022, - pidfile=pidfile, - files_preserve=[logger.handlers[0].stream] - ) - if args.user: - try: - context.uid = pwd.getpwnam(args.user).pw_uid - except KeyError: - logger.critical("Invalid user: %s" % args.user) - return 1 - # NOTE(LinuxJedi): we are switching user so need to switch - # the ownership of the log file for rotation - os.chown(logger.handlers[0].baseFilename, context.uid, -1) - if args.group: - try: - context.gid = grp.getgrnam(args.group).gr_gid - except KeyError: - logger.critical("Invalid group: %s" % args.group) - return 1 - - try: - context.open() - except lockfile.LockTimeout: - logger.critical( - "Failed to lock pidfile %s, another instance running?", - args.pid - ) - return 1 - - server.main() - return 0 diff --git a/libra/api/model/__init__.py b/libra/api/model/__init__.py new file mode 100644 index 00000000..554a28fe --- /dev/null +++ b/libra/api/model/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def init_model(): + """ + This is a stub method which is called at application startup time. + + If you need to bind to a parse database configuration, set up tables or + ORM classes, or perform any database initialization, this is the + recommended place to do it. + + For more information working with databases, and some common recipes, + see http://pecan.readthedocs.org/en/latest/databases.html + """ + pass diff --git a/libra/api/model/lbaas.py b/libra/api/model/lbaas.py new file mode 100644 index 00000000..f8b37b46 --- /dev/null +++ b/libra/api/model/lbaas.py @@ -0,0 +1,109 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Table, Column, Integer, ForeignKey, create_engine +from sqlalchemy import INTEGER, VARCHAR, TIMESTAMP, BIGINT +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship, backref, sessionmaker +from pecan import conf + +# TODO replace this with something better +conn_string = '''mysql://%s:%s@%s/%s''' % ( + conf.database.username, + conf.database.password, + conf.database.host, + conf.database.schema +) + +engine = create_engine(conn_string) +DeclarativeBase = declarative_base() +metadata = DeclarativeBase.metadata +metadata.bind = engine + +loadbalancers_devices = Table( + 'loadbalancers_devices', + metadata, + Column('loadbalancer', Integer, ForeignKey('loadbalancers.id')), + Column('device', Integer, ForeignKey('devices.id')) +) + + +class Device(DeclarativeBase): + """device model""" + __tablename__ = 'devices' + #column definitions + az = Column(u'az', INTEGER(), nullable=False) + created = Column(u'created', TIMESTAMP(), nullable=False) + floatingIpAddr = Column( + u'floatingIpAddr', VARCHAR(length=128), nullable=False + ) + id = Column(u'id', BIGINT(), primary_key=True, nullable=False) + name = Column(u'name', VARCHAR(length=128), nullable=False) + publicIpAddr = Column(u'publicIpAddr', VARCHAR(length=128), nullable=False) + status = Column(u'status', VARCHAR(length=128), nullable=False) + type = Column(u'type', VARCHAR(length=128), nullable=False) + updated = Column(u'updated', TIMESTAMP(), nullable=False) + + def find_free_device(self): + """queries for free and clear device + + sql form java api + SELECT * FROM devices WHERE loadbalancers = " + EMPTY_LBIDS + " AND + status = '" + Device.STATUS_OFFLINE + "'" ; + """ + return session.query(Device).\ + filter_by(loadbalancers="", status="OFFLINE").\ + first() + + +class LoadBalancer(DeclarativeBase): + """load balancer model""" + __tablename__ = 'loadbalancers' + #column definitions + algorithm = Column(u'algorithm', VARCHAR(length=80), nullable=False) + created = Column(u'created', TIMESTAMP(), nullable=False) + errmsg = Column(u'errmsg', VARCHAR(length=128)) + id = Column(u'id', BIGINT(), primary_key=True, nullable=False) + name = Column(u'name', VARCHAR(length=128), nullable=False) + port = Column(u'port', INTEGER(), nullable=False) + protocol = Column(u'protocol', VARCHAR(length=128), nullable=False) + status = Column(u'status', VARCHAR(length=50), nullable=False) + tenantid = Column(u'tenantid', VARCHAR(length=128), nullable=False) + updated = Column(u'updated', TIMESTAMP(), nullable=False) + nodes = relationship( + 'Node', backref=backref('loadbalancers', order_by='Node.id') + ) + devices = relationship( + 'Device', secondary=loadbalancers_devices, backref='loadbalancers', + lazy='joined' + ) + + +class Node(DeclarativeBase): + """node model""" + __tablename__ = 'nodes' + #column definitions + address = Column(u'address', VARCHAR(length=128), nullable=False) + enabled = Column(u'enabled', Integer(), nullable=False) + id = Column(u'id', BIGINT(), primary_key=True, nullable=False) + lbid = Column( + u'lbid', BIGINT(), ForeignKey('loadbalancers.id'), nullable=False + ) + port = Column(u'port', INTEGER(), nullable=False) + status = Column(u'status', VARCHAR(length=128), nullable=False) + weight = Column(u'weight', INTEGER(), nullable=False) + +"""session""" +session = sessionmaker(bind=engine)() diff --git a/libra/api/model/lbaas.sql b/libra/api/model/lbaas.sql new file mode 100644 index 00000000..d8cbc078 --- /dev/null +++ b/libra/api/model/lbaas.sql @@ -0,0 +1,66 @@ +# LBaaS Database schema +# pemellquist@gmail.com + +DROP DATABASE IF EXISTS lbaas; +CREATE DATABASE lbaas; +USE lbaas; + +# versions, used to define overall version for schema +# major version differences are not backward compatibile +create TABLE versions ( + major INT NOT NULL, + minor INT NOT NULL, + PRIMARY KEY (major) +); +INSERT INTO versions values (2,0); + +# loadbalancers +CREATE TABLE loadbalancers ( + id BIGINT NOT NULL AUTO_INCREMENT, # unique id for this loadbalancer, generated by DB when record is created + name VARCHAR(128) NOT NULL, # tenant assigned load balancer name + tenantid VARCHAR(128) NOT NULL, # tenant id who owns this loadbalancer + protocol VARCHAR(128) NOT NULL, # loadbalancer protocol used, can be 'HTTP', 'TCP' or 'HTTPS' + port INT NOT NULL, # TCP port number associated with protocol and used by loadbalancer northbound interface + status VARCHAR(50) NOT NULL, # current status, see ATLAS API 1.1 for all possible values + algorithm VARCHAR(80) NOT NULL, # LB Algorithm in use e.g. ROUND_ROBIN, see ATLAS API 1.1 for all possible values + created TIMESTAMP NOT NULL, # timestamp of when LB was created + updated TIMESTAMP NOT NULL, # timestamp of when LB was last updated + device BIGINT NOT NULL, # reference to associated device OR '0' for unassigned + errmsg VARCHAR(128), # optional error message which can describe details regarding LBs state, can be blank if no error state exists + PRIMARY KEY (id) # ids are unique accross all LBs + ) DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci; + + #nodes + CREATE TABLE nodes ( + id BIGINT NOT NULL AUTO_INCREMENT, # unique id for this node, generated by DB when record is created + lbid BIGINT NOT NULL, # Loadbalancer who owns this node + address VARCHAR(128) NOT NULL, # IPV4 or IPV6 address for this node + port INT NOT NULL, # TCP port number associated with this node and used from LB to node + weight INT NOT NULL, # Node weight if applicable to algorithm used + enabled BOOLEAN NOT NULL, # is node enabled or not + status VARCHAR(128) NOT NULL, # status of node 'OFFLINE', 'ONLINE', 'ERROR', this value is reported by the device + PRIMARY KEY (id) # ids are unique accross all Nodes + ) DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci; + + + # devices +CREATE TABLE devices ( + id BIGINT NOT NULL AUTO_INCREMENT, # unique id for this device, generated by DB when record is created + name VARCHAR(128) NOT NULL, # admin assigned device name, this is the unique gearman worker function name + floatingIpAddr VARCHAR(128) NOT NULL, # IPV4 or IPV6 address of device for floating IP + publicIpAddr VARCHAR(128) NOT NULL, # IPV4 or IPV6 address of device for floating IP + loadbalancers VARCHAR(128) NOT NULL, # Reference to loadbalancers using this device ( JSON array ) + az INT NOT NULL, # availability zone in which this device exists + type VARCHAR(128) NOT NULL, # text description of type of device, e.g. 'HAProxy' + created TIMESTAMP NOT NULL, # timestamp of when device was created + updated TIMESTAMP NOT NULL, # timestamp of when device was last updated + status VARCHAR(128) NOT NULL, # status of device 'OFFLINE', 'ONLINE', 'ERROR', this value is reported by the device + PRIMARY KEY (id) +) DEFAULT CHARSET utf8 DEFAULT COLLATE utf8_general_ci; + +CREATE TABLE `loadbalancers_devices` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `loadbalancer` int(11) DEFAULT NULL, + `device` int(11) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=16 DEFAULT CHARSET=latin1 diff --git a/libra/api/model/responses.py b/libra/api/model/responses.py new file mode 100644 index 00000000..4da48f51 --- /dev/null +++ b/libra/api/model/responses.py @@ -0,0 +1,296 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Class Responses +responder objects for framework. +""" + + +class Responses(object): + """404 - not found""" + _default = {'status': '404'} + + """not found """ + not_found = {'message': 'Object not Found'} + + """service_unavailable""" + service_unavailable = {'message': 'Service Unavailable'} + + """algorithms response""" + algorithms = { + 'algorithms': [ + {'name': 'ROUND_ROBIN'}, + {'name': 'LEAST_CONNECTIONS'} + ] + } + + """protocols response""" + protocols = { + 'protocols': [ + { + 'name': 'HTTP', + 'port': '80' + }, + { + 'name': 'HTTPS', + 'port': '443' + }, + { + 'name': 'TCP', + 'port': '*' + } + ] + } + + """class LoadBalancers + grouping of lb responses + """ + + class LoadBalancers(object): + """LoadBalancers list""" + get = { + 'loadBalancers': [ + { + 'name': 'lb-site1', + 'id': '71', + 'protocol': 'HTTP', + 'port': '80', + 'algorithm': 'LEAST_CONNECTIONS', + 'status': 'ACTIVE', + 'created': '2010-11-30T03:23:42Z', + 'updated': '2010-11-30T03:23:44Z' + }, + { + 'name': 'lb-site2', + 'id': '166', + 'protocol': 'TCP', + 'port': '9123', + 'algorithm': 'ROUND_ROBIN', + 'status': 'ACTIVE', + 'created': '2010-11-30T03:23:42Z', + 'updated': '2010-11-30T03:23:44Z' + } + ] + } + + """loadbalancer details""" + detail = { + 'id': '2000', + 'name': 'sample-loadbalancer', + 'protocol': 'HTTP', + 'port': '80', + 'algorithm': 'ROUND_ROBIN', + 'status': 'ACTIVE', + 'created': '2010-11-30T03:23:42Z', + 'updated': '2010-11-30T03:23:44Z', + 'virtualIps': [ + { + 'id': '1000', + 'address': '2001:cdba:0000:0000:0000:0000:3257:9652', + 'type': 'PUBLIC', + 'ipVersion': 'IPV6' + } + ], + 'nodes': [ + { + 'id': '1041', + 'address': '10.1.1.1', + 'port': '80', + 'condition': 'ENABLED', + 'status': 'ONLINE' + }, + { + 'id': '1411', + 'address': '10.1.1.2', + 'port': '80', + 'condition': 'ENABLED', + 'status': 'ONLINE' + } + ], + 'sessionPersistence': { + 'persistenceType': 'HTTP_COOKIE' + }, + 'connectionThrottle': { + 'maxRequestRate': '50', + 'rateInterval': '60' + } + } + + """create loadbalancer response""" + post = { + 'name': 'a-new-loadbalancer', + 'id': '144', + 'protocol': 'HTTP', + 'port': '83', + 'algorithm': 'ROUND_ROBIN', + 'status': 'BUILD', + 'created': '2011-04-13T14:18:07Z', + 'updated': '2011-04-13T14:18:07Z', + 'virtualIps': [ + { + 'address': '3ffe:1900:4545:3:200:f8ff:fe21:67cf', + 'id': '39', + 'type': 'PUBLIC', + 'ipVersion': 'IPV6' + } + ], + 'nodes': [ + { + 'address': '10.1.1.1', + 'id': '653', + 'port': '80', + 'status': 'ONLINE', + 'condition': 'ENABLED' + } + ] + } + + """virtualips""" + virtualips = { + 'virtualIps': [ + { + 'id': '1021', + 'address': '206.10.10.210', + 'type': 'PUBLIC', + 'ipVersion': 'IPV4' + } + ] + } + + """usage""" + usage = { + 'loadBalancerUsageRecords': [ + { + 'id': '394', + 'transferBytesIn': '2819204', + 'transferBytesOut': '84923069' + }, + { + 'id': '473', + 'transferBytesIn': '0', + 'transferBytesOut': '0' + } + ] + } + + """class HealthMonitor + monitor responses + """ + + class HealthMonitor(object): + """monitor CONNECT response""" + get = { + 'type': 'CONNECT', + 'delay': '20', + 'timeout': '10', + 'attemptsBeforeDeactivation': '3' + } + """monitor HTTPS response""" + get_https = { + 'type': 'HTTPS', + 'delay': '10', + 'timeout': '3', + 'attemptsBeforeDeactivation': '3', + 'path': '/healthcheck' + } + """class SessionPersistence + for managing Session Persistance + """ + + class SessionPersistence(object): + """get""" + get = { + 'persistenceType': 'HTTP_COOKIE' + } + """class Connections + Throttle Connections responses + """ + + class ConnectionThrottle(object): + """get""" + get = { + 'maxRequestRate': '50', + 'rateInterval': '60' + } + + """class Nodes + grouping of node related responses + """ + + class Nodes(object): + """list of nodes of a specific lb""" + get = { + 'nodes': [ + { + 'id': '410', + 'address': '10.1.1.1', + 'port': '80', + 'condition': 'ENABLED', + 'status': 'ONLINE' + }, + { + 'id': '236', + 'address': '10.1.1.2', + 'port': '80', + 'condition': 'ENABLED', + 'status': 'ONLINE' + }, + { + 'id': '2815', + 'address': '10.1.1.3', + 'port': '83', + 'condition': 'DISABLED', + 'status': 'OFFLINE' + }, + ] + } + + """a specific node details""" + get_detail = { + 'id': '236', + 'address': '10.1.1.2', + 'port': '80', + 'condition': 'ENABLED', + 'status': 'ONLINE' + } + + """nodes create response""" + post = { + 'nodes': [ + { + 'id': '7298', + 'address': '10.1.1.1', + 'port': '80', + 'condition': 'ENABLED', + 'status': 'ONLINE' + }, + { + 'id': '293', + 'address': '10.2.2.1', + 'port': '80', + 'weight': '2', + 'condition': 'ENABLED', + 'status': 'OFFLINE' + }, + { + 'id': '183', + 'address': '10.2.2.4', + 'port': '88', + 'weight': '2', + 'condition': 'DISABLED', + 'status': 'OFFLINE' + } + ] + } diff --git a/libra/api/model/validation.py b/libra/api/model/validation.py new file mode 100644 index 00000000..5c510fdd --- /dev/null +++ b/libra/api/model/validation.py @@ -0,0 +1,61 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Validation(object): + """class Validatoin + Validation templates for validict lib + """ + """loadbalancer_create""" + loadbalancer_create = { + "name": "a-new-loadbalancer", + "nodes": [ + { + "address": "10.1.1.1", + "port": "80" + }, + { + "address": "10.1.1.2", + "port": "81" + } + ] + } + """nodes_create""" + nodes_create = { + "nodes": [ + { + "address": "10.1.1.1", + "port": "80" + }, + { + "address": "10.2.2.1", + "port": "80", + "weight": "2" + }, + { + "address": "10.2.2.2", + "port": "88", + "condition": "DISABLED", + "weight": "2" + } + ] + } + """monitor CONNECT request""" + monitor_connect = { + "type": "CONNECT", + "delay": "20", + "timeout": "10", + "attemptsBeforeDeactivation": "3" + } diff --git a/libra/api/templates/error.html b/libra/api/templates/error.html new file mode 100644 index 00000000..f2d97961 --- /dev/null +++ b/libra/api/templates/error.html @@ -0,0 +1,12 @@ +<%inherit file="layout.html" /> + +## provide definitions for blocks we want to redefine +<%def name="title()"> + Server Error ${status} + + +## now define the body of the template +
+

Server Error ${status}

+
+

${message}

diff --git a/libra/api/tests/__init__.py b/libra/api/tests/__init__.py new file mode 100644 index 00000000..b7cc7196 --- /dev/null +++ b/libra/api/tests/__init__.py @@ -0,0 +1,36 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +from unittest import TestCase +from pecan import set_config +from pecan.testing import load_test_app + +__all__ = ['FunctionalTest'] + + +class FunctionalTest(TestCase): + """ + Used for functional tests where you need to test your + literal application and its integration with the framework. + """ + + def setUp(self): + self.app = load_test_app(os.path.join( + os.path.dirname(__file__), + 'config.py' + )) + + def tearDown(self): + set_config({}, overwrite=True) diff --git a/libra/api/tests/config.py b/libra/api/tests/config.py new file mode 100644 index 00000000..3b3ca392 --- /dev/null +++ b/libra/api/tests/config.py @@ -0,0 +1,50 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Server Specific Configurations +server = { + 'port': '8080', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'libra.api.controllers.root.RootController', + 'modules': ['libra.api'], + 'static_root': '%(confdir)s/../../public', + 'template_path': '%(confdir)s/../templates', + 'debug': True, + 'errors': { + '404': '/error/404', + '__force_dict__': True + } +} + +database = { + 'username':'root', + 'password':'', + 'host':'127.0.0.1', + 'schema':'lbaas' +} + +gearman = { + 'server':['localhost:4730'], +} + +# Custom Configurations must be in Python dictionary format:: +# +# foo = {'bar':'baz'} +# +# All configurations are accessible at:: +# pecan.conf diff --git a/libra/api/tests/test_functional.py b/libra/api/tests/test_functional.py new file mode 100644 index 00000000..f9b69880 --- /dev/null +++ b/libra/api/tests/test_functional.py @@ -0,0 +1,36 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +#from unittest import TestCase +#from webtest import TestApp +from libra.api.tests import FunctionalTest + + +class TestRootController(FunctionalTest): + + def test_get(self): + response = self.app.get('/') + assert response.status_int == 201 + + def test_search(self): + response = self.app.post('/', params={'q': 'RestController'}) + assert response.status_int == 201 +# assert response.headers['Location'] == ( +# 'http://pecan.readthedocs.org/en/latest/search.html' +# '?q=RestController' +# ) + + def test_get_not_found(self): + response = self.app.get('/a/bogus/url', expect_errors=True) +# assert response.status_int == 400 diff --git a/libra/api/tests/test_units.py b/libra/api/tests/test_units.py new file mode 100644 index 00000000..29a83f18 --- /dev/null +++ b/libra/api/tests/test_units.py @@ -0,0 +1,21 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import TestCase + + +class TestUnits(TestCase): + + def test_units(self): + assert 5 * 5 == 25 diff --git a/libra/openstack/common/jsonutils.py b/libra/openstack/common/jsonutils.py new file mode 100644 index 00000000..4a98da87 --- /dev/null +++ b/libra/openstack/common/jsonutils.py @@ -0,0 +1,167 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import functools +import inspect +import itertools +import json +import types +import xmlrpclib + +from libra.openstack.common import timeutils + + +_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + +_simple_types = (types.NoneType, int, basestring, bool, float, long) + + +def to_primitive(value, convert_instances=False, convert_datetime=True, + level=0, max_depth=3): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + # handle obvious types first - order of basic types determined by running + # full tests on nova project, resulting in the following counts: + # 572754 + # 460353 + # 379632 + # 274610 + # 199918 + # 114200 + # 51817 + # 26164 + # 6491 + # 283 + # 19 + if isinstance(value, _simple_types): + return value + + if isinstance(value, datetime.datetime): + if convert_datetime: + return timeutils.strtime(value) + else: + return value + + # value of itertools.count doesn't get caught by nasty_type_tests + # and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return unicode(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > max_depth: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + recursive = functools.partial(to_primitive, + convert_instances=convert_instances, + convert_datetime=convert_datetime, + level=level, + max_depth=max_depth) + if isinstance(value, dict): + return dict((k, recursive(v)) for k, v in value.iteritems()) + elif isinstance(value, (list, tuple)): + return [recursive(lv) for lv in value] + + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if convert_datetime and isinstance(value, datetime.datetime): + return timeutils.strtime(value) + elif hasattr(value, 'iteritems'): + return recursive(dict(value.iteritems()), level=level + 1) + elif hasattr(value, '__iter__'): + return recursive(list(value)) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return recursive(value.__dict__, level=level + 1) + else: + if any(test(value) for test in _nasty_type_tests): + return unicode(value) + return value + except TypeError: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/libra/openstack/common/timeutils.py b/libra/openstack/common/timeutils.py new file mode 100644 index 00000000..60943659 --- /dev/null +++ b/libra/openstack/common/timeutils.py @@ -0,0 +1,186 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Time related utilities and helper functions. +""" + +import calendar +import datetime + +import iso8601 + + +# ISO 8601 extended time format with microseconds +_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' +_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND + + +def isotime(at=None, subsecond=False): + """Stringify time in ISO 8601 format""" + if not at: + at = utcnow() + st = at.strftime(_ISO8601_TIME_FORMAT + if not subsecond + else _ISO8601_TIME_FORMAT_SUBSECOND) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + st += ('Z' if tz == 'UTC' else tz) + return st + + +def parse_isotime(timestr): + """Parse time from ISO 8601 format""" + try: + return iso8601.parse_date(timestr) + except iso8601.ParseError as e: + raise ValueError(e.message) + except TypeError as e: + raise ValueError(e.message) + + +def strtime(at=None, fmt=PERFECT_TIME_FORMAT): + """Returns formatted utcnow.""" + if not at: + at = utcnow() + return at.strftime(fmt) + + +def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): + """Turn a formatted time back into a datetime.""" + return datetime.datetime.strptime(timestr, fmt) + + +def normalize_time(timestamp): + """Normalize time in arbitrary timezone to UTC naive object""" + offset = timestamp.utcoffset() + if offset is None: + return timestamp + return timestamp.replace(tzinfo=None) - offset + + +def is_older_than(before, seconds): + """Return True if before is older than seconds.""" + if isinstance(before, basestring): + before = parse_strtime(before).replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) + + +def is_newer_than(after, seconds): + """Return True if after is newer than seconds.""" + if isinstance(after, basestring): + after = parse_strtime(after).replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + return calendar.timegm(utcnow().timetuple()) + + +def utcnow(): + """Overridable version of utils.utcnow.""" + if utcnow.override_time: + try: + return utcnow.override_time.pop(0) + except AttributeError: + return utcnow.override_time + return datetime.datetime.utcnow() + + +def iso8601_from_timestamp(timestamp): + """Returns a iso8601 formated date from timestamp""" + return isotime(datetime.datetime.utcfromtimestamp(timestamp)) + + +utcnow.override_time = None + + +def set_time_override(override_time=datetime.datetime.utcnow()): + """ + Override utils.utcnow to return a constant time or a list thereof, + one at a time. + """ + utcnow.override_time = override_time + + +def advance_time_delta(timedelta): + """Advance overridden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + try: + for dt in utcnow.override_time: + dt += timedelta + except TypeError: + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overridden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + +def marshall_now(now=None): + """Make an rpc-safe datetime with microseconds. + + Note: tzinfo is stripped, but not required for relative times.""" + if not now: + now = utcnow() + return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, + minute=now.minute, second=now.second, + microsecond=now.microsecond) + + +def unmarshall_time(tyme): + """Unmarshall a datetime dict.""" + return datetime.datetime(day=tyme['day'], + month=tyme['month'], + year=tyme['year'], + hour=tyme['hour'], + minute=tyme['minute'], + second=tyme['second'], + microsecond=tyme['microsecond']) + + +def delta_seconds(before, after): + """ + Compute the difference in seconds between two date, time, or + datetime objects (as a float, to microsecond resolution). + """ + delta = after - before + try: + return delta.total_seconds() + except AttributeError: + return ((delta.days * 24 * 3600) + delta.seconds + + float(delta.microseconds) / (10 ** 6)) + + +def is_soon(dt, window): + """ + Determines if time is going to happen in the next window seconds. + + :params dt: the time + :params window: minimum seconds to remain to consider the time not soon + + :return: True if expiration is within the given duration + """ + soon = (utcnow() + datetime.timedelta(seconds=window)) + return normalize_time(dt) <= soon diff --git a/libra/openstack/common/xmlutils.py b/libra/openstack/common/xmlutils.py new file mode 100644 index 00000000..b131d3e2 --- /dev/null +++ b/libra/openstack/common/xmlutils.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from xml.dom import minidom +from xml.parsers import expat +from xml import sax +from xml.sax import expatreader + + +class ProtectedExpatParser(expatreader.ExpatParser): + """An expat parser which disables DTD's and entities by default.""" + + def __init__(self, forbid_dtd=True, forbid_entities=True, + *args, **kwargs): + # Python 2.x old style class + expatreader.ExpatParser.__init__(self, *args, **kwargs) + self.forbid_dtd = forbid_dtd + self.forbid_entities = forbid_entities + + def start_doctype_decl(self, name, sysid, pubid, has_internal_subset): + raise ValueError("Inline DTD forbidden") + + def entity_decl(self, entityName, is_parameter_entity, value, base, + systemId, publicId, notationName): + raise ValueError(" entity declaration forbidden") + + def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name): + # expat 1.2 + raise ValueError(" unparsed entity forbidden") + + def external_entity_ref(self, context, base, systemId, publicId): + raise ValueError(" external entity forbidden") + + def notation_decl(self, name, base, sysid, pubid): + raise ValueError(" notation forbidden") + + def reset(self): + expatreader.ExpatParser.reset(self) + if self.forbid_dtd: + self._parser.StartDoctypeDeclHandler = self.start_doctype_decl + self._parser.EndDoctypeDeclHandler = None + if self.forbid_entities: + self._parser.EntityDeclHandler = self.entity_decl + self._parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl + self._parser.ExternalEntityRefHandler = self.external_entity_ref + self._parser.NotationDeclHandler = self.notation_decl + try: + self._parser.SkippedEntityHandler = None + except AttributeError: + # some pyexpat versions do not support SkippedEntity + pass + + +def safe_minidom_parse_string(xml_string): + """Parse an XML string using minidom safely. + + """ + try: + return minidom.parseString(xml_string, parser=ProtectedExpatParser()) + except sax.SAXParseException: + raise expat.ExpatError() diff --git a/openstack-common.conf b/openstack-common.conf index 76914cf5..6bb6996f 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,7 +1,7 @@ [DEFAULT] # The list of modules to copy from openstack-common -modules=importutils +modules=importutils,jsonutils,xmlutils # The base module to hold the copy of openstack.common base=libra diff --git a/requirements.txt b/requirements.txt index 08b9775b..9c7a4e3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,6 @@ python_novaclient>=2.11.1 python_swiftclient>=1.3.0 requests>=1.0.0 dogapi +pecan +sqlalchemy +MySQL-python