From da7805282c1c1d3f85a3a4f744438b7b63f3d0cc Mon Sep 17 00:00:00 2001 From: Rudi Schlatte Date: Thu, 4 Apr 2024 13:54:08 +0200 Subject: [PATCH] Use clamped requirements for broker Go through the resource broker for node candidates again. Ask for >= x, <= 2*x when we have requirement x (for cpu, memory). This gives us some flexibility in case no image precisely fulfills the requirements, but rules out nodes that are outrageously big. Change-Id: I35d0b4207b2b76b212f5e584f932b8aaf579a0e9 --- .../optimiser/kubevela/KubevelaAnalyzer.java | 235 +++++++++++++----- .../controller/NebulousAppDeployer.java | 12 +- .../controller/NebulousAppTests.java | 4 +- 3 files changed, 186 insertions(+), 65 deletions(-) diff --git a/nebulous-requirements-extractor/src/main/java/eu/nebulouscloud/optimiser/kubevela/KubevelaAnalyzer.java b/nebulous-requirements-extractor/src/main/java/eu/nebulouscloud/optimiser/kubevela/KubevelaAnalyzer.java index fdb62d0..f142c50 100644 --- a/nebulous-requirements-extractor/src/main/java/eu/nebulouscloud/optimiser/kubevela/KubevelaAnalyzer.java +++ b/nebulous-requirements-extractor/src/main/java/eu/nebulouscloud/optimiser/kubevela/KubevelaAnalyzer.java @@ -75,18 +75,99 @@ public class KubevelaAnalyzer { return getNodeCount(parseKubevela(kubevela)); } + /** + * Add frequirements for Ubuntu version 22.04. Also add requirement for + * 2GB of RAM for now until we know more about the size / cpu requirements + * of the nebulous runtime. + * + * @param reqs The list of requirements to add to. + */ + private static void addNebulousRequirements(List reqs) { + reqs.add(new AttributeRequirement("image", "operatingSystem.family", + RequirementOperator.IN, OperatingSystemFamily.UBUNTU.toString())); + reqs.add(new AttributeRequirement("image", "name", RequirementOperator.INC, "22")); + reqs.add(new AttributeRequirement("hardware", "ram", RequirementOperator.GEQ, "2048")); + } + + /** + * Get cpu requirement, taken from "cpu" resource requirement in KubeVela + * and rounding up to nearest whole number. + * + * @param c A Component branch of the parsed KubeVela file. + * @param componentName the component name, used only for logging. + * @return an integer of number of cores required, or -1 in case of no + * requirement. + */ + private static long getCpuRequirement(JsonNode c, String componentName) { + JsonNode cpu = c.at("/properties/cpu"); + if (cpu.isMissingNode()) cpu = c.at("/properties/resources/requests/cpu"); + if (!cpu.isMissingNode()) { + // KubeVela has fractional core /cpu requirements, and the + // value might be given as a string instead of a number, so + // parse string in all cases. + double kubevela_cpu = -1; + try { + kubevela_cpu = Double.parseDouble(cpu.asText()); + } catch (NumberFormatException e) { + log.warn("CPU spec in " + componentName + " is not a number, value seen is " + cpu.asText()); + return -1; + } + long sal_cores = Math.round(Math.ceil(kubevela_cpu)); + if (sal_cores > 0) { + return sal_cores; + } else { + // floatValue returns 0.0 if node is not numeric + log.warn("CPU spec in " + componentName + " is not a number, value seen is " + cpu.asText()); + return -1; + } + } else { + // no spec given + return -1; + } + } + + /** + * Get memory requirement, taken from "memory" resource requirement in KubeVela + * and converted to Megabytes. We currently handle the "Mi" and "Gi" + * suffixes that KubeVela uses. + * + * @param c A Component branch of the parsed KubeVela file. + * @param componentName the component name, used only for logging. + * @return an integer of memory required in Mb, or -1 in case of no + * requirement. + */ + private static long getMemoryRequirement(JsonNode c, String componentName) { + JsonNode memory = c.at("/properties/memory"); + if (memory.isMissingNode()) memory = c.at("/properties/resources/requests/memory"); + if (!memory.isMissingNode()) { + long sal_memory = -1; + String sal_memory_str = memory.asText(); + if (sal_memory_str.endsWith("Mi")) { + sal_memory = Long.parseLong(sal_memory_str.substring(0, sal_memory_str.length() - 2)); + } else if (sal_memory_str.endsWith("Gi")) { + sal_memory = Long.parseLong(sal_memory_str.substring(0, sal_memory_str.length() - 2)) * 1024; + } else { + log.warn("Unsupported memory specification in component " + componentName + " : " + memory.asText() + " (wanted 'Mi' or 'Gi') "); + } + return sal_memory; + } else { + return -1; + } + } + + /** * Extract node requirements from a KubeVela file in a form we can send to * the SAL `findNodeCandidates` endpoint.

* * We read the following attributes for each component: * - * - `properties.cpu`, `properties.requests.cpu`: round up to next integer - * and generate requirement `hardware.cores` + * - `properties.cpu`, `properties.resources.requests.cpu`: round up to + * next integer and generate requirement `hardware.cores` * - * - `properties.memory`, `properties.requests.memory`: Handle "200Mi", - * "0.2Gi" and bare number, convert to MB and generate requirement - * `hardware.memory` + * - `properties.memory`, `properties.resources.requests.memory`: Handle + * "200Mi", "0.2Gi" and bare number, convert to MB and generate + * requirement `hardware.memory` * * Notes:

* @@ -111,60 +192,24 @@ public class KubevelaAnalyzer { * family) list of requirements for that component. No requirements mean * any node will suffice. */ - public static Map> getRequirements(JsonNode kubevela, boolean includeNebulousRequirements) { + public static Map> getBoundedRequirements(JsonNode kubevela, boolean includeNebulousRequirements) { Map> result = new HashMap<>(); ArrayNode components = kubevela.withArray("/spec/components"); for (final JsonNode c : components) { String componentName = c.get("name").asText(); ArrayList reqs = new ArrayList<>(); if (includeNebulousRequirements) { - // We want Ubuntu, version 22.04, and 2GB of RAM until we know - // more about the size / cpu requirements of the nebulous - // runtime. - reqs.add(new AttributeRequirement("image", "operatingSystem.family", - RequirementOperator.IN, OperatingSystemFamily.UBUNTU.toString())); - reqs.add(new AttributeRequirement("image", "name", RequirementOperator.INC, "22")); - reqs.add(new AttributeRequirement("hardware", "ram", RequirementOperator.GEQ, "2048")); + addNebulousRequirements(reqs); } - JsonNode cpu = c.at("/properties/cpu"); - if (cpu.isMissingNode()) cpu = c.at("/properties/resources/requests/cpu"); - if (!cpu.isMissingNode()) { - // KubeVela has fractional core /cpu requirements, and the - // value might be given as a string instead of a number, so - // parse string in all cases. - double kubevela_cpu = -1; - try { - kubevela_cpu = Double.parseDouble(cpu.asText()); - } catch (NumberFormatException e) { - log.warn("CPU spec in " + componentName + " is not a number, value seen is " + cpu.asText()); - } - long sal_cores = Math.round(Math.ceil(kubevela_cpu)); - if (sal_cores > 0) { - reqs.add(new AttributeRequirement("hardware", "cores", - RequirementOperator.GEQ, Long.toString(sal_cores))); - } else { - // floatValue returns 0.0 if node is not numeric - log.warn("CPU spec in " + componentName + " is not a number, value seen is " + cpu.asText()); - } + long cores = getCpuRequirement(c, componentName); + if (cores > 0) { + reqs.add(new AttributeRequirement("hardware", "cores", + RequirementOperator.GEQ, Long.toString(cores))); } - JsonNode memory = c.at("/properties/memory"); - if (memory.isMissingNode()) cpu = c.at("/properties/resources/requests/memory"); - if (!memory.isMissingNode()) { - String sal_memory = memory.asText(); - if (sal_memory.endsWith("Mi")) { - sal_memory = sal_memory.substring(0, sal_memory.length() - 2); - } else if (sal_memory.endsWith("Gi")) { - sal_memory = String.valueOf(Integer.parseInt(sal_memory.substring(0, sal_memory.length() - 2)) * 1024); - } else if (!memory.isNumber()) { - log.warn("Unsupported memory specification in component " + componentName + " : " + memory.asText() + " (wanted 'Mi' or 'Gi') "); - sal_memory = null; - } - // Fall-through: we rewrote the KubeVela file and didn't add - // the "Mi" suffix, but it's a number - if (sal_memory != null) { + long memory = getMemoryRequirement(c, componentName); + if (memory > 0) { reqs.add(new AttributeRequirement("hardware", "ram", - RequirementOperator.GEQ, sal_memory)); - } + RequirementOperator.GEQ, Long.toString(memory))); } for (final JsonNode t : c.withArray("/traits")) { // TODO: Check for node affinity / geoLocation / country / @@ -178,27 +223,103 @@ public class KubevelaAnalyzer { /** * Get node requirements for app components, including nebulous-specific - * requirements. This method calls {@link #getRequirements(JsonNode, + * requirements. This method calls {@link #getBoundedRequirements(JsonNode, * boolean)} with second parameter {@code true}. * - * @see #getRequirements(JsonNode, boolean) + * @see #getBoundedRequirements(JsonNode, boolean) */ - public static Map> getRequirements(JsonNode kubevela) { - return getRequirements(kubevela, true); + public static Map> getBoundedRequirements(JsonNode kubevela) { + return getBoundedRequirements(kubevela, true); + } + + /** + * Get node requirements for app components, including nebulous-specific + * requirements. Like {@link #getBoundedRequirements} but also include an + * upper bound of twice the requirement size. I.e., for cpu=2, we ask for + * cpu >= 2, cpu <= 4. Take care to not ask for less than 2048Mb of + * memory since that's the minimum Nebulous requirement for now. + */ + public static Map> getClampedRequirements(JsonNode kubevela) { + Map> result = new HashMap<>(); + ArrayNode components = kubevela.withArray("/spec/components"); + for (final JsonNode c : components) { + String componentName = c.get("name").asText(); + ArrayList reqs = new ArrayList<>(); + addNebulousRequirements(reqs); + long cores = getCpuRequirement(c, componentName); + if (cores > 0) { + reqs.add(new AttributeRequirement("hardware", "cores", + RequirementOperator.GEQ, Long.toString(cores))); + reqs.add(new AttributeRequirement("hardware", "cores", + RequirementOperator.LEQ, Long.toString(cores * 2))); + } + long memory = getMemoryRequirement(c, componentName); + if (memory > 0) { + reqs.add(new AttributeRequirement("hardware", "ram", + RequirementOperator.GEQ, Long.toString(memory))); + reqs.add(new AttributeRequirement("hardware", "ram", + // See addNebulousRequirements(), don't ask for both more + // and less than 2048 + RequirementOperator.LEQ, Long.toString(Math.max(memory * 2, 2048)))); + } + for (final JsonNode t : c.withArray("/traits")) { + // TODO: Check for node affinity / geoLocation / country / + // node type (edge or cloud) + } + // Finally, add requirements for this job to the map + result.put(componentName, reqs); + } + return result; + } + + /** + * Get node requirements for app components, including nebulous-specific + * requirements. Like {@link #getBoundedRequirements} but require precise + * amounts, i.e., ask for precisely cpu == 2, memory == 2048 instead of + * asking for >= or <=. Note that we still ask for >= 2048 Mb since + * that's the nebulous lower bound for now. + */ + public static Map> getPreciseRequirements(JsonNode kubevela) { + Map> result = new HashMap<>(); + ArrayNode components = kubevela.withArray("/spec/components"); + for (final JsonNode c : components) { + String componentName = c.get("name").asText(); + ArrayList reqs = new ArrayList<>(); + addNebulousRequirements(reqs); + long cores = getCpuRequirement(c, componentName); + if (cores > 0) { + reqs.add(new AttributeRequirement("hardware", "cores", + RequirementOperator.EQ, Long.toString(cores))); + } + long memory = getMemoryRequirement(c, componentName); + if (memory > 0) { + reqs.add(new AttributeRequirement("hardware", "ram", + // See addNebulousRequirements; don't ask for less than + // the other constraint allows + RequirementOperator.EQ, Long.toString(Math.max(memory, 2048)))); + } + for (final JsonNode t : c.withArray("/traits")) { + // TODO: Check for node affinity / geoLocation / country / + // node type (edge or cloud) + } + // Finally, add requirements for this job to the map + result.put(componentName, reqs); + } + return result; } /** * Extract node requirements from a KubeVela file. * - * @see #getRequirements(JsonNode) + * @see #getBoundedRequirements(JsonNode) * @param kubevela The KubeVela file, as a YAML string. * @return a map of component name to (potentially empty, except for OS * family) list of requirements for that component. No requirements mean * any node will suffice. * @throws JsonProcessingException if kubevela does not contain valid YAML. */ - public static Map> getRequirements(String kubevela) throws JsonProcessingException { - return getRequirements(parseKubevela(kubevela)); + public static Map> getBoundedRequirements(String kubevela) throws JsonProcessingException { + return getBoundedRequirements(parseKubevela(kubevela)); } /** diff --git a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java index eb03c3d..2681fe3 100644 --- a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java +++ b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java @@ -165,7 +165,7 @@ public class NebulousAppDeployer { // ------------------------------------------------------------ // 1. Extract node requirements - Map> componentRequirements = KubevelaAnalyzer.getRequirements(kubevela); + Map> componentRequirements = KubevelaAnalyzer.getClampedRequirements(kubevela); Map nodeCounts = KubevelaAnalyzer.getNodeCount(kubevela); List controllerRequirements = getControllerRequirements(appUUID); @@ -177,7 +177,7 @@ public class NebulousAppDeployer { // 2. Find node candidates // TODO: filter by app resources (check enabled: true in resources array) - List controllerCandidates = conn.findNodeCandidatesFromSal(controllerRequirements, appUUID); + List controllerCandidates = conn.findNodeCandidates(controllerRequirements, appUUID); if (controllerCandidates.isEmpty()) { log.error("Could not find node candidates for requirements: {}", controllerRequirements, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); @@ -189,7 +189,7 @@ public class NebulousAppDeployer { String nodeName = e.getKey(); List requirements = e.getValue(); // TODO: filter by app resources (check enabled: true in resources array) - List candidates = conn.findNodeCandidatesFromSal(requirements, appUUID); + List candidates = conn.findNodeCandidates(requirements, appUUID); if (candidates.isEmpty()) { log.error("Could not find node candidates for for node {}, requirements: {}", nodeName, requirements, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); @@ -415,7 +415,7 @@ public class NebulousAppDeployer { // ------------------------------------------------------------ // 1. Extract node requirements - Map> componentRequirements = KubevelaAnalyzer.getRequirements(kubevela); + Map> componentRequirements = KubevelaAnalyzer.getClampedRequirements(kubevela); Map componentReplicaCounts = KubevelaAnalyzer.getNodeCount(kubevela); Map> oldComponentRequirements = app.getComponentRequirements(); @@ -444,7 +444,7 @@ public class NebulousAppDeployer { log.debug("Adding {} nodes to component {}", nAdd, componentName, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); // TODO: filter by app resources (check enabled: true in resources array) - List candidates = conn.findNodeCandidatesFromSal(newR, appUUID); + List candidates = conn.findNodeCandidates(newR, appUUID); if (candidates.isEmpty()) { log.error("Could not find node candidates for requirements: {}", newR, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); @@ -500,7 +500,7 @@ public class NebulousAppDeployer { log.debug("Redeploying all nodes of component {}", componentName, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); // TODO: filter by app resources (check enabled: true in resources array) - List candidates = conn.findNodeCandidatesFromSal(newR, appUUID); + List candidates = conn.findNodeCandidates(newR, appUUID); if (candidates.size() == 0) { log.error("Empty node candidate list for component {}, continuing without creating node", componentName, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); diff --git a/optimiser-controller/src/test/java/eu/nebulouscloud/optimiser/controller/NebulousAppTests.java b/optimiser-controller/src/test/java/eu/nebulouscloud/optimiser/controller/NebulousAppTests.java index a31b87c..f5479aa 100644 --- a/optimiser-controller/src/test/java/eu/nebulouscloud/optimiser/controller/NebulousAppTests.java +++ b/optimiser-controller/src/test/java/eu/nebulouscloud/optimiser/controller/NebulousAppTests.java @@ -91,7 +91,7 @@ public class NebulousAppTests { String kubevela_str = Files.readString(getResourcePath("vela-deployment-v2.yml"), StandardCharsets.UTF_8); JsonNode kubevela = yaml_mapper.readTree(kubevela_str); - Map> requirements = KubevelaAnalyzer.getRequirements(kubevela); + Map> requirements = KubevelaAnalyzer.getBoundedRequirements(kubevela); // We could compare the requirements with what is contained in // KubeVela, or compare keys with component names, but this would // essentially duplicate the method code--so we just make sure the @@ -111,7 +111,7 @@ public class NebulousAppTests { ObjectNode replacements = solutions.withObject("VariableValues"); ObjectNode kubevela1 = app.rewriteKubevelaWithSolution(replacements); - Map> requirements = KubevelaAnalyzer.getRequirements(kubevela1); + Map> requirements = KubevelaAnalyzer.getBoundedRequirements(kubevela1); // We could compare the requirements with what is contained in // KubeVela, or compare keys with component names, but this would // essentially duplicate the method code--so we just make sure the