From 2fe692328dc6405df4e28525e571f842f49e900f Mon Sep 17 00:00:00 2001 From: Rudi Schlatte Date: Fri, 23 Feb 2024 14:22:31 +0100 Subject: [PATCH] Ask resource broker for node candidates Change-Id: If8bf869790e3ed565dbf985c2caba23864f4c264 --- .../optimiser/controller/ExnConnector.java | 9 +- .../controller/NebulousAppDeployer.java | 28 +- .../optimiser/controller/SalConnector.java | 40 +- .../src/test/resources/CFBS_ResponseV2.json | 504 ++++++++++++++++++ .../src/test/resources/README.md | 3 + 5 files changed, 557 insertions(+), 27 deletions(-) create mode 100644 optimiser-controller/src/test/resources/CFBS_ResponseV2.json diff --git a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java index 6e4fe0c..b8bce1e 100644 --- a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java +++ b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java @@ -75,10 +75,15 @@ public class ExnConnector { public static final SyncedPublisher createJob = new SyncedPublisher("createJob", "eu.nebulouscloud.exn.sal.job.post", true, true); - /** The findNodeCandidates endpoint. */ + /** The findNodeCandidates endpoint. Should not be used during normal + * operation--ask the broker instead. */ public static final SyncedPublisher findNodeCandidates = new SyncedPublisher("findNodeCandidates", "eu.nebulouscloud.exn.sal.nodecandidate.get", true, true); + /** The findNodeCandidates endpoint (Broker's version). */ + public static final SyncedPublisher findBrokerNodeCandidates + = new SyncedPublisher("findBrokerNodeCandidates", + "eu.nebulouscloud.cfsb.get_node_candidates", true, true); /** The addNodes endpoint. */ public static final SyncedPublisher addNodes = new SyncedPublisher("addNodes", @@ -107,7 +112,7 @@ public class ExnConnector { callback, // List.of(new Publisher("config", "config", true)), List.of(amplMessagePublisher, - createJob, findNodeCandidates, addNodes, submitJob), + createJob, findNodeCandidates, findBrokerNodeCandidates, addNodes, submitJob), List.of( new Consumer("ui_app_messages", app_creation_channel, new AppCreationMessageHandler(), true, true), diff --git a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java index 0d02e7f..ef98b74 100644 --- a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java +++ b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java @@ -51,7 +51,8 @@ public class NebulousAppDeployer { @Getter private static CommandsInstallation controllerInstallation = new CommandsInstallation(); - private static final ObjectMapper yaml_mapper = new ObjectMapper(new YAMLFactory()); + private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + private static final ObjectMapper mapper = new ObjectMapper(); // TODO: find out the commands to initialize the workers /** @@ -289,19 +290,18 @@ public class NebulousAppDeployer { // ---------------------------------------- // 2. Find node candidates - // TODO: switch to asking the cloud broker for candidates when it's - // ready - List controllerCandidates = SalConnector.findNodeCandidates(controllerRequirements, appUUID); + ArrayNode controllerCandidates = SalConnector.findNodeCandidates(controllerRequirements, appUUID); if (controllerCandidates.isEmpty()) { - log.error("Could not find node candidates for requirements: {}", controllerRequirements); + log.error("Could not find node candidates for requirements: {}", + controllerRequirements, keyValue("appId", appUUID)); // Continue here while we don't really deploy // return; } - Map> workerCandidates = new HashMap<>(); + Map workerCandidates = new HashMap<>(); for (Map.Entry> e : workerRequirements.entrySet()) { String nodeName = e.getKey(); List requirements = e.getValue(); - List candidates = SalConnector.findNodeCandidates(requirements, appUUID); + ArrayNode candidates = SalConnector.findNodeCandidates(requirements, appUUID); if (candidates.isEmpty()) { log.error("Could not find node candidates for requirements: {}", requirements); // Continue here while we don't really deploy @@ -313,7 +313,7 @@ public class NebulousAppDeployer { // ------------------------------------------------------------ // 3. Select node candidates - log.debug("Collecting worker nodes for {}", appUUID); + log.debug("Collecting worker nodes for {}", appUUID, keyValue("appId", appUUID)); Map nodeNameToCandidate = new HashMap<>(); for (Map.Entry> e : workerRequirements.entrySet()) { // Here we collect two things: the flat list (hostname -> @@ -325,6 +325,9 @@ public class NebulousAppDeployer { for (int i = 1; i <= numberOfNodes; i++) { String nodeName = String.format("%s-%s", componentName, i); nodeNames.add(nodeName); + // TODO: choose the node candidate with the highest score + // and/or ranking. + // TODO: Here we need to discriminate between edge and cloud // node candidates: we can deploy an edge node only once, but // cloud nodes arbitrarily often. So if the best node @@ -336,8 +339,11 @@ public class NebulousAppDeployer { if (!workerCandidates.get(componentName).isEmpty()) { // should always be true, except currently we don't abort // in Step 2 if we don't find candidates. - NodeCandidate candidate = workerCandidates.get(componentName).get(0); - nodeNameToCandidate.put(nodeName, candidate); + JsonNode candidate = workerCandidates.get(componentName).get(0); + NodeCandidate c = mapper.convertValue(((ObjectNode)candidate).deepCopy() + .remove(List.of("score", "ranking")), + NodeCandidate.class); + nodeNameToCandidate.put(nodeName, c); } } app.getComponentMachineNames().put(componentName, nodeNames); @@ -359,7 +365,7 @@ public class NebulousAppDeployer { JsonNode rewritten = addNodeAffinities(kubevela, app.getComponentMachineNames()); String rewritten_kubevela = "---\n# Did not manage to create rewritten KubeVela"; try { - rewritten_kubevela = yaml_mapper.writeValueAsString(rewritten); + rewritten_kubevela = yamlMapper.writeValueAsString(rewritten); } catch (JsonProcessingException e) { log.error("Failed to convert KubeVela to YAML; this should never happen", e); } diff --git a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/SalConnector.java b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/SalConnector.java index 6139fcd..1500740 100644 --- a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/SalConnector.java +++ b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/SalConnector.java @@ -10,6 +10,8 @@ import org.ow2.proactive.sal.model.Requirement; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import static net.logstash.logback.argument.StructuredArguments.keyValue; @@ -25,18 +27,27 @@ import static net.logstash.logback.argument.StructuredArguments.keyValue; @Slf4j public class SalConnector { + private SalConnector() {} + private static final ObjectMapper mapper = new ObjectMapper(); /** - * Get list of node candidates that fulfil the requirements. + * Get list of node candidates from the resource broker that fulfil the + given requirements. * - * See https://github.com/ow2-proactive/scheduling-abstraction-layer/blob/master/documentation/nodecandidates-endpoints.md#71--filter-node-candidates-endpoint + *

Note that we cannot convert the result to a list containing {@code + * org.ow2.proactive.sal.model.NodeCandidate} instances, since the broker + * adds the additional fields {@code score} and {@code ranking}. Instead + * we return a JSON {@code ArrayNode} containing {@code ObjectNode}s in + * the format specified at + * https://github.com/ow2-proactive/scheduling-abstraction-layer/blob/master/documentation/nodecandidates-endpoints.md#71--filter-node-candidates-endpoint + * but with these two additional attributes. * * @param requirements The list of requirements. - * @param appID The application ID, if available. - * @return A list of node candidates, or null in case of error. + * @param appID The application ID. + * @return A JSON array containing node candidates. */ - public static List findNodeCandidates(List requirements, String appID) { + public static ArrayNode findNodeCandidates(List requirements, String appID) { Map msg; try { msg = Map.of( @@ -47,15 +58,16 @@ public class SalConnector { keyValue("appId", appID), e); return null; } - Map response = ExnConnector.findNodeCandidates.sendSync(msg, appID, null, false); - String body = response.get("body").toString(); // body is a string already - try { - return Arrays.asList(mapper.readValue(body, NodeCandidate[].class)); - } catch (JsonProcessingException e) { - log.error("Error receiving findNodeCandidates result (this should never happen)", - keyValue("appId", appID), e); - return null; - } + Map response = ExnConnector.findBrokerNodeCandidates.sendSync(msg, appID, null, false); + ObjectNode jsonBody = mapper.convertValue(response, ObjectNode.class); + // Note: what we would really like to do here is something like: + // + // return Arrays.asList(mapper.readValue(response, NodeCandidate[].class)); + // + // But since the broker adds two attributes, the array elements cannot + // be deserialized into org.ow2.proactive.sal.model.NodeCandidate + // objects. + return jsonBody.withArray("/nodes"); } } diff --git a/optimiser-controller/src/test/resources/CFBS_ResponseV2.json b/optimiser-controller/src/test/resources/CFBS_ResponseV2.json new file mode 100644 index 0000000..a16c777 --- /dev/null +++ b/optimiser-controller/src/test/resources/CFBS_ResponseV2.json @@ -0,0 +1,504 @@ +{ + "nodes": [ + { + "id": "1b2a77f4-0f81-4910-a15c-0dd57c6a89ff", + "nodeCandidateType": "EDGE", + "jobIdForByon": null, + "jobIdForEdge": null, + "price": 0.0558, + "cloud": { + "id": "nebulous-aws-sal-1", + "endpoint": null, + "cloudType": "PUBLIC", + "api": { + "providerName": "aws-ec2" + }, + "credential": null, + "cloudConfiguration": { + "nodeGroup": "", + "properties": {} + }, + "owner": null, + "state": null, + "diagnostic": null + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 47.4705, + "longitude": 6.1918 + }, + "parent": null, + "state": null, + "owner": null + }, + "image": { + "id": "", + "name": "PrEstoCloud-Golden-Image", + "providerId": "", + "operatingSystem": { + "operatingSystemFamily": "UNKNOWN_OS_FAMILY", + "operatingSystemArchitecture": "I386", + "operatingSystemVersion": 0.0 + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "hardware": { + "id": "nebulous-aws-sal-1/eu-west-2/t3.medium", + "name": "", + "providerId": "", + "cores": 2, + "ram": 4096, + "disk": 16.0, + "fpga": 0, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "pricePerInvocation": 0.0, + "memoryPrice": 0.0, + "nodeId": "", + "environment": null, + "score": 1, + "rank": 1 + }, + { + "id": "93b846d4-3f43-4e4b-87f2-c3ba07bb5555", + "nodeCandidateType": "EDGE", + "jobIdForByon": null, + "jobIdForEdge": null, + "price": 0.0125, + "cloud": { + "id": "nebulous-aws-sal-1", + "endpoint": null, + "cloudType": "PUBLIC", + "api": { + "providerName": "aws-ec2" + }, + "credential": null, + "cloudConfiguration": { + "nodeGroup": "", + "properties": {} + }, + "owner": null, + "state": null, + "diagnostic": null + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 47.4705, + "longitude": 6.1918 + }, + "parent": null, + "state": null, + "owner": null + }, + "image": { + "id": "", + "name": "PrEstoCloud-Golden-Image", + "providerId": "", + "operatingSystem": { + "operatingSystemFamily": "UNKNOWN_OS_FAMILY", + "operatingSystemArchitecture": "I386", + "operatingSystemVersion": 0.0 + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "hardware": { + "id": "nebulous-aws-sal-1/eu-west-2/t3.medium", + "name": "", + "providerId": "", + "cores": 2, + "ram": 4096, + "disk": 16.0, + "fpga": 0, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "pricePerInvocation": 0.0, + "memoryPrice": 0.0, + "nodeId": "", + "environment": null, + "score": 0.938, + "rank": 2 + }, + { + "id": "3b53f32d-53ca-410e-969b-6bf542a3fd32", + "nodeCandidateType": "EDGE", + "jobIdForByon": null, + "jobIdForEdge": null, + "price": 0.0836, + "cloud": { + "id": "nebulous-aws-sal-1", + "endpoint": null, + "cloudType": "PUBLIC", + "api": { + "providerName": "aws-ec2" + }, + "credential": null, + "cloudConfiguration": { + "nodeGroup": "", + "properties": {} + }, + "owner": null, + "state": null, + "diagnostic": null + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 47.4705, + "longitude": 6.1918 + }, + "parent": null, + "state": null, + "owner": null + }, + "image": { + "id": "", + "name": "PrEstoCloud-Golden-Image", + "providerId": "", + "operatingSystem": { + "operatingSystemFamily": "UNKNOWN_OS_FAMILY", + "operatingSystemArchitecture": "I386", + "operatingSystemVersion": 0.0 + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "hardware": { + "id": "nebulous-aws-sal-1/eu-west-2/t3.medium", + "name": "", + "providerId": "", + "cores": 2, + "ram": 4096, + "disk": 16.0, + "fpga": 0, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "pricePerInvocation": 0.0, + "memoryPrice": 0.0, + "nodeId": "", + "environment": null, + "score": 0.786, + "rank": 3 + }, + { + "id": "39b00b2a-d430-4557-aed0-b7fa758d83db", + "nodeCandidateType": "EDGE", + "jobIdForByon": null, + "jobIdForEdge": null, + "price": 0.0703, + "cloud": { + "id": "nebulous-aws-sal-1", + "endpoint": null, + "cloudType": "PUBLIC", + "api": { + "providerName": "aws-ec2" + }, + "credential": null, + "cloudConfiguration": { + "nodeGroup": "", + "properties": {} + }, + "owner": null, + "state": null, + "diagnostic": null + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 47.4705, + "longitude": 6.1918 + }, + "parent": null, + "state": null, + "owner": null + }, + "image": { + "id": "", + "name": "PrEstoCloud-Golden-Image", + "providerId": "", + "operatingSystem": { + "operatingSystemFamily": "UNKNOWN_OS_FAMILY", + "operatingSystemArchitecture": "I386", + "operatingSystemVersion": 0.0 + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "hardware": { + "id": "nebulous-aws-sal-1/eu-west-2/t3.medium", + "name": "", + "providerId": "", + "cores": 2, + "ram": 4096, + "disk": 16.0, + "fpga": 0, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "pricePerInvocation": 0.0, + "memoryPrice": 0.0, + "nodeId": "", + "environment": null, + "score": 0.764, + "rank": 4 + }, + { + "id": "4b64de40-053a-4ab7-bf68-55a82938507d", + "nodeCandidateType": "EDGE", + "jobIdForByon": null, + "jobIdForEdge": null, + "price": 0.0122, + "cloud": { + "id": "nebulous-aws-sal-1", + "endpoint": null, + "cloudType": "PUBLIC", + "api": { + "providerName": "aws-ec2" + }, + "credential": null, + "cloudConfiguration": { + "nodeGroup": "", + "properties": {} + }, + "owner": null, + "state": null, + "diagnostic": null + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 47.4705, + "longitude": 6.1918 + }, + "parent": null, + "state": null, + "owner": null + }, + "image": { + "id": "", + "name": "PrEstoCloud-Golden-Image", + "providerId": "", + "operatingSystem": { + "operatingSystemFamily": "UNKNOWN_OS_FAMILY", + "operatingSystemArchitecture": "I386", + "operatingSystemVersion": 0.0 + }, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "hardware": { + "id": "nebulous-aws-sal-1/eu-west-2/t3.medium", + "name": "", + "providerId": "", + "cores": 2, + "ram": 4096, + "disk": 16.0, + "fpga": 0, + "location": { + "id": "", + "name": "", + "providerId": "", + "locationScope": "REGION", + "isAssignable": true, + "geoLocation": { + "city": "Paris", + "country": "France", + "latitude": 48.8607, + "longitude": 2.3281 + }, + "parent": null, + "state": null, + "owner": null + }, + "state": null, + "owner": null + }, + "pricePerInvocation": 0.0, + "memoryPrice": 0.0, + "nodeId": "", + "environment": null, + "score": 0.661, + "rank": 5 + } + ] +} \ No newline at end of file diff --git a/optimiser-controller/src/test/resources/README.md b/optimiser-controller/src/test/resources/README.md index 6012e87..d8c8320 100644 --- a/optimiser-controller/src/test/resources/README.md +++ b/optimiser-controller/src/test/resources/README.md @@ -4,3 +4,6 @@ on 2023-10-24. The canonical `vela-deployment.yaml` file is probably at https://gitlab.ubitech.eu/nebulous/use-cases/surveillance-dsl-demo/ + + +- `CFBS_ResponseV2.json`: node candidate response as sent by the resource broker. (2024-02-23)