From d7fd5a85a772341260c7e43496612699383d4c73 Mon Sep 17 00:00:00 2001 From: Lisa Zangrando Date: Mon, 13 Jun 2016 09:50:02 +0200 Subject: [PATCH] Dynamic quota initialization fixed! - Dynamic quota initialization fixed but it is needed a mechanism for restoring the original project quota - added --long option to 'get_share' shell command Change-Id: Ib31840e7b1700fc36c06ad6352c629412ff91ec7 --- synergy_scheduler_manager/client/command.py | 88 +++++++++++++------ .../fairshare_manager.py | 2 + synergy_scheduler_manager/nova_manager.py | 14 +-- synergy_scheduler_manager/quota_manager.py | 47 ++++------ .../scheduler_manager.py | 25 ++++-- 5 files changed, 102 insertions(+), 74 deletions(-) diff --git a/synergy_scheduler_manager/client/command.py b/synergy_scheduler_manager/client/command.py index d6e8490..b3ba47a 100644 --- a/synergy_scheduler_manager/client/command.py +++ b/synergy_scheduler_manager/client/command.py @@ -217,11 +217,17 @@ class GetShare(Execute): super(GetShare, self).__init__("GET_SHARE") def configureParser(self, subparser): - subparser.add_parser("get_share", - add_help=True, - help="shows the users share") + parser = subparser.add_parser("get_share", + add_help=True, + help="shows the users share") + + parser.add_argument("--long", + action='store_true', + help="shows more details") def sendRequest(self, synergy_url, args): + self.long = args.long + super(GetShare, self).sendRequest( synergy_url + "/synergy/execute", "FairShareManager", @@ -233,41 +239,65 @@ class GetShare(Execute): max_prj = len("project") max_usr = len("user") max_prj_share = len("share") - max_usr_share = len("share (abs)") + max_usr_share = len("share") - for project in projects.values(): - max_prj = max(len(project["name"]), max_prj) - max_prj_share = max(len("{:.2f}".format(project["share"])), - max_prj_share) + if self.long: + for project in projects.values(): + max_prj = max(len(project["name"]), max_prj) + max_prj_share = max(len("{:.2f}% ({:.2f})".format( + project["norm_share"] * 100, project["share"])), + max_prj_share) - for user in project["users"].values(): - max_usr = max(len(user["name"]), max_usr) - max_usr_share = max( - len("{:.2f} ({:.2f})".format(user["share"], - user["norm_share"])), - max_usr_share) + for user in project["users"].values(): + max_usr = max(len(user["name"]), max_usr) + max_usr_share = max( + len("{:.2f}%".format(user["norm_share"] * 100)), + max_usr_share) - separator_str = "-" * (max_prj + max_usr + max_prj_share + + separator = "-" * (max_prj + max_usr + max_prj_share + max_usr_share + 13) + "\n" - data_str = "| {0:%ss} | {1:%ss} | {2:%ss} | {3:%ss} |\n" % ( - max_prj, max_prj_share, max_usr, max_usr_share) + raw = "| {0:%ss} | {1:%ss} | {2:%ss} | {3:%ss} |\n" % ( + max_prj, max_prj_share, max_usr, max_usr_share) - msg = separator_str - msg += data_str.format("project", "share", "user", "share (abs)") - msg += separator_str + msg = separator + msg += raw.format("project", "share", "user", "share") + msg += separator - for project in projects.values(): - for user in project["users"].values(): - msg += data_str.format( + for project in projects.values(): + for user in project["users"].values(): + msg += raw.format( + project["name"], + "{:.2f}% ({:.2f})".format(project["norm_share"] * 100, + project["share"]), + user["name"], + "{:.2f}%".format(user["norm_share"] * 100)) + + msg += separator + print(msg) + else: + for project in projects.values(): + max_prj = max(len(project["name"]), max_prj) + max_prj_share = max(len("{:.2f}% ({:.2f})".format( + project["norm_share"] * 100, project["share"])), + max_prj_share) + + separator = "-" * (max_prj + max_prj_share + 7) + "\n" + + raw = "| {0:%ss} | {1:%ss} |\n" % (max_prj, max_prj_share) + + msg = separator + msg += raw.format("project", "share") + msg += separator + + for project in projects.values(): + msg += raw.format( project["name"], - "{:.2f}".format(project["share"]), - user["name"], - "{:.2f} ({:.2f})".format(user["share"], - user["norm_share"])) + "{:.2f}% ({:.2f})".format(project["norm_share"] * 100, + project["share"])) - msg += separator_str - print(msg) + msg += separator + print(msg) class GetUsage(Execute): diff --git a/synergy_scheduler_manager/fairshare_manager.py b/synergy_scheduler_manager/fairshare_manager.py index 47ff961..2444226 100644 --- a/synergy_scheduler_manager/fairshare_manager.py +++ b/synergy_scheduler_manager/fairshare_manager.py @@ -289,8 +289,10 @@ class FairShareManager(Manager): if prj_share > 0 and sibling_share > 0 and total_prj_share > 0: user["norm_share"] = (user_share / sibling_share) * \ (prj_share / total_prj_share) + project["norm_share"] = prj_share / total_prj_share else: user["norm_share"] = user_share + project["norm_share"] = prj_share if total_usage_ram > 0: user_usage["norm_ram"] /= total_usage_ram diff --git a/synergy_scheduler_manager/nova_manager.py b/synergy_scheduler_manager/nova_manager.py index 0abf000..fe390ba 100644 --- a/synergy_scheduler_manager/nova_manager.py +++ b/synergy_scheduler_manager/nova_manager.py @@ -853,9 +853,11 @@ class NovaManager(Manager): return response_data - def updateQuota(self, id, data): + def updateQuota(self, id, cores, ram, instances): url = "os-quota-sets/%s" % id - quota_set = {"quota_set": data} + quota_set = {"quota_set": {"cores": cores, + "ram": ram, + "instances": instances}} try: response_data = self.getResource(url, "PUT", quota_set) @@ -936,13 +938,6 @@ class NovaManager(Manager): return self.messagingAPI.getNotificationListener(targets, endpoints) def getResourceUsage(self, prj_ids, from_date, to_date): - # LOG.info("getUsage: fromDate=%s period_length=%s days" - # % (fromDate, period_length)) - # print("getUsage: fromDate=%s period_length=%s days" - # % (fromDate, period_length)) - - # period = str(period_length) - resource_usage = {} connection = self.db_engine.connect() @@ -982,7 +977,6 @@ terminated_at is NULL) group by user_id, project_id\ # for row in result.fetchall(): for row in result: - # LOG.info("row=%s" % row) user_id = row[0] prj_id = row[1] diff --git a/synergy_scheduler_manager/quota_manager.py b/synergy_scheduler_manager/quota_manager.py index 4bc40f6..a0d19e4 100644 --- a/synergy_scheduler_manager/quota_manager.py +++ b/synergy_scheduler_manager/quota_manager.py @@ -75,6 +75,9 @@ class DynamicQuota(object): self.projects[prj_id] = project self.condition.notifyAll() + else: + raise Exception("project %r (id=%s) alredy added!" + % (prj_name, prj_id)) def removeProject(self, prj_id): if prj_id in self.projects: @@ -271,13 +274,13 @@ class QuotaManager(Manager): return self.dynamic_quota.getProject(prj_id) def addProject(self, prj_id, prj_name): - try: - quota = {"cores": -1, "ram": -1, "instances": -1} - self.nova_manager.execute("UPDATE_QUOTA", prj_id, quota) + if self.dynamic_quota.getProject(prj_id) is not None: + raise Exception("project %r (id=%s) alredy added!" + % (prj_name, prj_id)) + try: usage = self.nova_manager.execute("GET_PROJECT_USAGE", prj_id) self.dynamic_quota.addProject(prj_id, prj_name, usage) - self.updateDynamicQuota() except Exception as ex: LOG.error(ex) @@ -297,9 +300,6 @@ class QuotaManager(Manager): for instance_id in ids: self.nova_manager.execute("DELETE_SERVER", instance_id) - quota = self.nova_manager.execute("GET_QUOTA", defaults=True) - self.nova_manager.execute("UPDATE_QUOTA", prj_id, quota) - self.dynamic_quota.removeProject(prj_id) self.updateDynamicQuota() @@ -345,9 +345,6 @@ class QuotaManager(Manager): name="ram_allocation_ratio", default=float(1.5)) - quota_default = self.nova_manager.execute("GET_QUOTA", - defaults=True) - hypervisors = self.nova_manager.execute("GET_HYPERVISORS") for hypervisor in hypervisors: @@ -370,37 +367,29 @@ class QuotaManager(Manager): if self.dynamic_quota.getProject(prj_id) is None: quota = self.nova_manager.execute("GET_QUOTA", prj_id) - if quota["cores"] == -1 and quota["ram"] == -1: - quota["cores"] = quota_default["cores"] - quota["ram"] = quota_default["ram"] - - try: - self.nova_manager.execute("UPDATE_QUOTA", - prj_id, - quota_default) - except Exception as ex: - LOG.error(ex) - static_cores += quota["cores"] static_ram += quota["ram"] enabled = False if total_cores < static_cores: - LOG.warn("dynamic quota: the total statically " - "allocated cores (%s) is greater than the total " - "amount of cores allowed (%s)" - % (static_cores, total_cores)) + if self.dynamic_quota.getProjects(): + LOG.warn("dynamic quota: the total statically " + "allocated cores (%s) is greater than the " + "total amount of cores allowed (%s)" + % (static_cores, total_cores)) else: enabled = True dynamic_cores = total_cores - static_cores if total_ram < static_ram: enabled = False - LOG.warn("dynamic quota: the total statically " - "allocated ram (%s) is greater than the total " - "amount of ram allowed (%s)" - % (static_ram, total_ram)) + + if self.dynamic_quota.getProjects(): + LOG.warn("dynamic quota: the total statically " + "allocated ram (%s) is greater than the " + "total amount of ram allowed (%s)" + % (static_ram, total_ram)) else: enabled = True dynamic_ram = total_ram - static_ram diff --git a/synergy_scheduler_manager/scheduler_manager.py b/synergy_scheduler_manager/scheduler_manager.py index b011980..1e9202d 100644 --- a/synergy_scheduler_manager/scheduler_manager.py +++ b/synergy_scheduler_manager/scheduler_manager.py @@ -286,6 +286,8 @@ class SchedulerManager(Manager): self.dynamic_quota = self.quota_manager.execute( "GET_DYNAMIC_QUOTA") + defaults = self.nova_manager.execute("GET_QUOTA", defaults=True) + k_projects = self.keystone_manager.execute("GET_PROJECTS") for k_project in k_projects: @@ -301,6 +303,23 @@ class SchedulerManager(Manager): "share": float(0), "TTL": self.default_TTL} + self.nova_manager.execute("UPDATE_QUOTA", + id=prj_id, + cores=-1, + ram=-1, + instances=-1) + else: + quota = self.nova_manager.execute("GET_QUOTA", id=prj_id) + + if quota["cores"] == -1 and quota["ram"] == -1 and \ + quota["instances"] == -1: + self.nova_manager.execute( + "UPDATE_QUOTA", + id=prj_id, + cores=defaults["cores"], + ram=defaults["ram"], + instances=defaults["instances"]) + if len(CONF.SchedulerManager.projects) > 0: raise Exception("projects %s not found" % CONF.SchedulerManager.projects) @@ -321,12 +340,6 @@ class SchedulerManager(Manager): del self.projects[prj_name] self.projects[prj_id] = project - quota = {"cores": -1, "ram": -1, "instances": -1} - - self.nova_manager.execute("UPDATE_QUOTA", - id=prj_id, - data=quota) - self.quota_manager.execute("ADD_PROJECT", prj_id=prj_id, prj_name=prj_name)