diff --git a/libra/api/app.py b/libra/api/app.py index 59fa0e75..361b9efd 100644 --- a/libra/api/app.py +++ b/libra/api/app.py @@ -78,6 +78,10 @@ class LogStdout(object): if data.strip() != '': self.logger(data) + # Gearman calls this + def flush(self): + pass + def main(): options = Options('api', 'API Server') diff --git a/libra/api/controllers/load_balancers.py b/libra/api/controllers/load_balancers.py index b51b7e7e..d4f60926 100644 --- a/libra/api/controllers/load_balancers.py +++ b/libra/api/controllers/load_balancers.py @@ -59,13 +59,14 @@ class LoadBalancersController(RestController): LoadBalancer.port, LoadBalancer.algorithm, LoadBalancer.status, LoadBalancer.created, LoadBalancer.updated - ).filter_by(tenantid=tenant_id).all()} + ).filter(LoadBalancer.tenantid == tenant_id). + filter(LoadBalancer.status != 'DELETED').all()} else: load_balancers = session.query( LoadBalancer.name, LoadBalancer.id, LoadBalancer.protocol, LoadBalancer.port, LoadBalancer.algorithm, LoadBalancer.status, LoadBalancer.created, - LoadBalancer.updated + LoadBalancer.updated, LoadBalancer.statusDescription ).join(LoadBalancer.devices).\ filter(LoadBalancer.tenantid == tenant_id).\ filter(LoadBalancer.id == load_balancer_id).\ @@ -145,7 +146,8 @@ class LoadBalancersController(RestController): nodelimit = session.query(Limits.value).\ filter(Limits.name == 'maxNodesPerLoadBalancer').scalar() count = session.query(LoadBalancer).\ - filter(LoadBalancer.tenantid == tenant_id).count() + filter(LoadBalancer.tenantid == tenant_id).\ + filter(LoadBalancer.status != 'DELETED').count() # TODO: this should probably be a 413, not sure how to do that yet if count >= lblimit: @@ -304,7 +306,7 @@ class LoadBalancersController(RestController): raise ClientSideError(errstr) @expose('json') - def delete(self, load_balancer_id, status=202): + def delete(self, load_balancer_id): """Remove a load balancer from the account. :param load_balancer_id: id of lb @@ -322,7 +324,8 @@ class LoadBalancersController(RestController): # grab the lb lb = session.query(LoadBalancer).\ filter(LoadBalancer.id == load_balancer_id).\ - filter(LoadBalancer.tenantid == tenant_id).first() + filter(LoadBalancer.tenantid == tenant_id).\ + filter(LoadBalancer.status != 'DELETED').first() if lb is None: response.status = 400 @@ -346,6 +349,7 @@ class LoadBalancersController(RestController): submit_job( 'DELETE', device.name, device.id, lb.id ) + response.status = 202 return None except: logger = logging.getLogger(__name__) diff --git a/libra/api/controllers/nodes.py b/libra/api/controllers/nodes.py index 15d5ec23..dd69fbf5 100644 --- a/libra/api/controllers/nodes.py +++ b/libra/api/controllers/nodes.py @@ -58,6 +58,7 @@ class NodesController(RestController): ).join(LoadBalancer.nodes).\ filter(LoadBalancer.tenantid == tenant_id).\ filter(LoadBalancer.id == self.lbid).\ + filter(LoadBalancer.status != 'DELETED').\ all() node_response = {'nodes': []} @@ -104,7 +105,6 @@ class NodesController(RestController): Returns: dict of the full list of nodes or the details of the single node """ - # TODO: gearman message tenant_id = get_limited_to_project(request.headers) if self.lbid is None: raise ClientSideError('Load Balancer ID has not been supplied') @@ -115,6 +115,7 @@ class NodesController(RestController): load_balancer = session.query(LoadBalancer).\ filter(LoadBalancer.tenantid == tenant_id).\ filter(LoadBalancer.id == self.lbid).\ + filter(LoadBalancer.status != 'DELETED').\ first() if load_balancer is None: raise ClientSideError('Load Balancer not found') @@ -177,7 +178,6 @@ class NodesController(RestController): Returns: None """ - # TODO: gearman message tenant_id = get_limited_to_project(request.headers) if self.lbid is None: response.status = 400 @@ -190,6 +190,7 @@ class NodesController(RestController): load_balancer = session.query(LoadBalancer).\ filter(LoadBalancer.tenantid == tenant_id).\ filter(LoadBalancer.id == self.lbid).\ + filter(LoadBalancer.device != 'DELETED').\ first() if load_balancer is None: response.status = 400 @@ -227,5 +228,5 @@ class NodesController(RestController): submit_job( 'UPDATE', device.name, device.id, self.lbid ) - + response.status = 202 return None diff --git a/libra/api/library/gearman_client.py b/libra/api/library/gearman_client.py index 5d8799a8..9bf02aef 100644 --- a/libra/api/library/gearman_client.py +++ b/libra/api/library/gearman_client.py @@ -58,6 +58,7 @@ class GearmanClientThread(object): LoadBalancer ).join(LoadBalancer.devices).\ filter(Device.id == data).\ + filter(LoadBalancer.status != 'DELETED').\ count() if count >= 1: # This is an update message because we want to retain the @@ -66,6 +67,7 @@ class GearmanClientThread(object): join(LoadBalancer.devices).\ filter(Device.id == data).\ filter(LoadBalancer.id != self.lbid).\ + filter(LoadBalancer.status != 'DELETED').\ first() job_data = { 'hpcs_action': 'UPDATE', @@ -95,22 +97,39 @@ class GearmanClientThread(object): filter(LoadBalancer.id == self.lbid).\ first() if not status: - lb.status = 'ERROR' - lb.errmsg = response + self._set_error(data, response) else: lb.status = 'DELETED' if count == 0: + # Device should never be used again device = session.query(Device).\ filter(Device.id == data).first() - device.status = 'OFFLINE' + device.status = 'DELETED' session.commit() + def _set_error(self, device_id, errmsg): + lbs = session.query( + LoadBalancer + ).join(LoadBalancer.nodes).\ + join(LoadBalancer.devices).\ + filter(Device.id == device_id).\ + filter(LoadBalancer.status != 'DELETED').\ + all() + device = session.query(Device).\ + filter(Device.id == device_id).\ + first() + device.status = 'ERROR' + for lb in lbs: + lb.status = 'ERROR' + lb.errmsg = errmsg + def send_update(self, data): lbs = session.query( LoadBalancer ).join(LoadBalancer.nodes).\ join(LoadBalancer.devices).\ filter(Device.id == data).\ + filter(LoadBalancer.status != 'DELETED').\ all() job_data = { 'hpcs_action': 'UPDATE', @@ -139,12 +158,9 @@ class GearmanClientThread(object): filter(LoadBalancer.id == self.lbid).\ first() if not status: - lb.status = 'ERROR' - lb.errmsg = response - pass + self._set_error(data, response) else: lb.status = 'ACTIVE' - pass session.commit() def _send_message(self, message): @@ -154,9 +170,13 @@ class GearmanClientThread(object): ) if job_status.state == 'UNKNOWN': # Gearman server connection failed + self.logger.error('Could not talk to gearman server') return False, "System error communicating with load balancer" if job_status.timed_out: # Job timed out + self.logger.warning( + 'Gearman timeout talking to {0}'.format(self.host) + ) return False, "Timeout error communicating with load balancer" self.logger.debug(job_status.result) if 'badRequest' in job_status.result: diff --git a/libra/api/model/lbaas.py b/libra/api/model/lbaas.py index 778549ac..8172b5f8 100644 --- a/libra/api/model/lbaas.py +++ b/libra/api/model/lbaas.py @@ -85,6 +85,7 @@ class LoadBalancer(DeclarativeBase): port = Column(u'port', INTEGER(), nullable=False) protocol = Column(u'protocol', VARCHAR(length=128), nullable=False) status = Column(u'status', VARCHAR(length=50), nullable=False) + statusDescription = Column(u'errmsg', VARCHAR(length=128), nullable=True) tenantid = Column(u'tenantid', VARCHAR(length=128), nullable=False) updated = Column(u'updated', FormatedDateTime(), nullable=False) created = Column(u'created', FormatedDateTime(), nullable=False)