From a7d04758197d5114c52533f80d6e9ec796f3800e Mon Sep 17 00:00:00 2001 From: Lisa Zangrando Date: Thu, 16 Jun 2016 16:53:39 +0200 Subject: [PATCH] getExpiredServers() fixed - NovaManager.getExpiredServers(): fixed DB query for retrieving the servers to be deleted - SchedulerManager.task(): updated due to the NovaManager.getExpiredServers() fix + some logging info - QuotaManager: updated due to the NovaManager.getExpiredServers() fix Change-Id: Ib48ddbc6b9f87c616481e41b5dfda77aa2aa015d --- synergy_scheduler_manager/nova_manager.py | 27 +++++++-------- .../scheduler_manager.py | 34 +++++++++---------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/synergy_scheduler_manager/nova_manager.py b/synergy_scheduler_manager/nova_manager.py index fe390ba..1202e92 100644 --- a/synergy_scheduler_manager/nova_manager.py +++ b/synergy_scheduler_manager/nova_manager.py @@ -1007,8 +1007,8 @@ terminated_at is NULL) group by user_id, project_id\ # retrieve the amount of resources in terms of cores # and ram the specified project is consuming QUERY = """select uuid, vcpus, memory_mb from nova.instances \ -where project_id='%(project_id)s' and deleted_at is NULL and (vm_state in \ -('error') or (vm_state in ('active') and terminated_at is NULL))\ +where project_id='%(project_id)s' and deleted_at is NULL and (vm_state=\ +'error' or (vm_state='active' and terminated_at is NULL))\ """ % {"project_id": prj_id} LOG.debug("getProjectUsage query: %s" % QUERY) @@ -1027,7 +1027,7 @@ where project_id='%(project_id)s' and deleted_at is NULL and (vm_state in \ return usage def getExpiredServers(self, prj_id, instances, TTL): - uuids = [] + servers = {} connection = self.db_engine.connect() try: @@ -1035,25 +1035,22 @@ where project_id='%(project_id)s' and deleted_at is NULL and (vm_state in \ # project and expiration time ids = "'%s'" % "', '".join(instances) - QUERY = """select uuid from nova.instances where project_id = \ -'%(project_id)s' and deleted_at is NULL and (vm_state in ('error') or \ -(uuid in (%(instances)s) and ((vm_state in ('active') and terminated_at is \ -NULL and timestampdiff(minute, launched_at, utc_timestamp()) >= \ -%(expiration)s) or (vm_state in ('building') and task_state in ('scheduling') \ -and created_at != updated_at and timestampdiff(minute, updated_at, \ -utc_timestamp()) >= 20))))""" % {"project_id": prj_id, - "instances": ids, - "expiration": TTL} + QUERY = """select uuid, vm_state from nova.instances where project_id = \ +'%(project_id)s' and deleted_at is NULL and (vm_state='error' or \ +(uuid in (%(instances)s) and vm_state='active' and terminated_at is NULL \ +and timestampdiff(minute, launched_at, utc_timestamp()) >= %(expiration)s))\ +""" % {"project_id": prj_id, "instances": ids, "expiration": TTL} - LOG.debug("getProjectUsage query: %s" % QUERY) + LOG.debug("getExpiredServers query: %s" % QUERY) result = connection.execute(QUERY) for row in result.fetchall(): - uuids.append(row[0]) + servers[row[0]] = row[1] + except SQLAlchemyError as ex: raise Exception(ex.message) finally: connection.close() - return uuids + return servers diff --git a/synergy_scheduler_manager/scheduler_manager.py b/synergy_scheduler_manager/scheduler_manager.py index 1e9202d..65df8e5 100644 --- a/synergy_scheduler_manager/scheduler_manager.py +++ b/synergy_scheduler_manager/scheduler_manager.py @@ -154,20 +154,12 @@ class Worker(threading.Thread): % (self.name, uuid, ex)) self.queue.deleteItem(queue_item) - self.quota.release(instance_id=uuid, - prj_id=prj_id, - cores=vcpus, - ram=memory_mb) + self.nova_manager.execute("DELETE_SERVER", id=uuid) continue if server["OS-EXT-STS:vm_state"] != "building" or \ server["OS-EXT-STS:task_state"] != "scheduling": self.queue.deleteItem(queue_item) - - self.quota.release(instance_id=uuid, - prj_id=prj_id, - cores=vcpus, - ram=memory_mb) continue if self.quota.allocate(instance_id=uuid, @@ -386,14 +378,22 @@ class SchedulerManager(Manager): for prj_id, project in self.dynamic_quota.getProjects().items(): instances = project["instances"]["active"] TTL = self.projects[prj_id]["TTL"] - uuids = self.nova_manager.execute("GET_EXPIRED_SERVERS", - prj_id=prj_id, - instances=instances, - TTL=TTL) + servers = self.nova_manager.execute("GET_EXPIRED_SERVERS", + prj_id=prj_id, + instances=instances, + TTL=TTL) + + for uuid, state in servers.items(): + if state == "error": + LOG.info("the server instance %r will be destroyed because" + " it is in %s state (TTL=%s, prj_id=%r)" + % (uuid, state, TTL, prj_id)) + else: + LOG.info("the server instance %r will be destroyed because" + " it exceeded its maximum time to live (TTL=%s, " + "state=%s, prj_id=%r)" + % (uuid, TTL, state, prj_id)) - for uuid in uuids: - LOG.info("deleting the expired instance %r from project=%s" - % (uuid, prj_id)) self.nova_manager.execute("DELETE_SERVER", id=uuid) def destroy(self): @@ -426,7 +426,7 @@ class SchedulerManager(Manager): prj_id=prj_id, cores=vcpus, ram=memory_mb) - priority = 999999999999 + priority = 99999999 LOG.info("released resource uuid %s " "num_attempts %s" % (uuid, num_attempts)) except Exception as ex: