diff --git a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java index c076be2..c57bc2b 100644 --- a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java +++ b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/ExnConnector.java @@ -113,7 +113,7 @@ public class ExnConnector { findSalNodeCandidates = new SyncedPublisher("findSalNodeCandidates", "eu.nebulouscloud.exn.sal.nodecandidate.get", true, true); findBrokerNodeCandidates = new SyncedPublisher("findBrokerNodeCandidates", "eu.nebulouscloud.cfsb.get_node_candidates", true, true); defineCluster = new SyncedPublisher("defineCluster", "eu.nebulouscloud.exn.sal.cluster.define", true, true); - getCluster = new SyncedPublisher("getCluster", "eu.nebulouscloud.exn.sal.cluster", true, true); + getCluster = new SyncedPublisher("getCluster", "eu.nebulouscloud.exn.sal.cluster.get", true, true); labelNodes = new SyncedPublisher("labelNodes", "eu.nebulouscloud.exn.sal.cluster.label", true, true); deployCluster = new SyncedPublisher("deployCluster", "eu.nebulouscloud.exn.sal.cluster.deploy", true, true); deployApplication = new SyncedPublisher("deployApplication", "eu.nebulouscloud.exn.sal.cluster.deployapplication", true, true); @@ -441,13 +441,13 @@ public class ExnConnector { /** * Get the definition of a cluster created by {@link #defineCluster}. * - * @param appID The application ID, as used to define the cluster. + * @param clusterName The cluster name, as given in {@link defineCluster}. * @return The cluster definition, or null in case of error. */ - public JsonNode getCluster(String appID) { - Map msg = Map.of("metaData", Map.of("user", "admin", "clusterName", appID)); - Map response = getCluster.sendSync(msg, appID, null, false); - JsonNode payload = extractPayloadFromExnResponse(response, appID, "getCluster"); + public JsonNode getCluster(String clusterName) { + Map msg = Map.of("metaData", Map.of("user", "admin", "clusterName", clusterName)); + Map response = getCluster.sendSync(msg, clusterName, null, false); + JsonNode payload = extractPayloadFromExnResponse(response, clusterName, "getCluster"); return payload.isMissingNode() ? null : payload; } diff --git a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java index a9063f4..2848d87 100644 --- a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java +++ b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/NebulousAppDeployer.java @@ -12,8 +12,6 @@ import eu.nebulouscloud.optimiser.kubevela.KubevelaAnalyzer; import org.ow2.proactive.sal.model.AttributeRequirement; import org.ow2.proactive.sal.model.NodeCandidate; import org.ow2.proactive.sal.model.NodeCandidate.NodeCandidateTypeEnum; -import org.ow2.proactive.sal.model.NodeType; -import org.ow2.proactive.sal.model.NodeTypeRequirement; import org.ow2.proactive.sal.model.OperatingSystemFamily; import org.ow2.proactive.sal.model.Requirement; import org.ow2.proactive.sal.model.RequirementOperator; @@ -103,7 +101,26 @@ public class NebulousAppDeployer { * @return a fresh node name. */ private static String createNodeName(String clusterName, String componentName, int deployGeneration, int nodeNumber) { - return String.format("%s-%s-%s-%s", clusterName, componentName, deployGeneration, nodeNumber); + return String.format("N%s-%s-%s-%s", clusterName, componentName, deployGeneration, nodeNumber); + } + + /** + * Given a cluster definition (as returned by {@link + * ExnConnector#getCluster}), return if all nodes are ready, i.e., are in + * state {@code "Finished"}. Once this method returns {@code true}, it is + * safe to call {@link ExnConnector#labelNodes} and {@link + * ExnConnector#deployApplication}. + * + * @param clusterStatus The cluster status, as returned by {@link + * ExnConnector#getCluster}. + * @return {@code true} if all nodes are in state {@code "Finished"}, + * {@code false} otherwise. + */ + private static boolean isClusterDeploymentFinished(JsonNode clusterStatus) { + return clusterStatus.withArray("/nodes") + .findParents("state") + .stream() + .allMatch(node -> node.get("state").asText().equals("Finished")); } /** @@ -123,7 +140,6 @@ public class NebulousAppDeployer { String appUUID = app.getUUID(); String clusterName = app.getClusterName(); ExnConnector conn = app.getExnConnector(); - Map edgeCandidates = app.getNodeEdgeCandidates(); log.info("Starting initial deployment for application", keyValue("appId", appUUID), keyValue("clusterName", clusterName)); int deployGeneration = app.getDeployGeneration() + 1; @@ -133,7 +149,7 @@ public class NebulousAppDeployer { // // 1. Extract node requirements and node counts from the KubeVela // definition. - // 2. Ask resource broker for node candidates for all workers and the + // 2. Ask resource broker for node candidates for all components and the // controller. // 3. Select node candidates, making sure to only select edge nodes // once. @@ -146,12 +162,12 @@ public class NebulousAppDeployer { // ------------------------------------------------------------ // 1. Extract node requirements - Map> workerRequirements = KubevelaAnalyzer.getRequirements(kubevela); + Map> componentRequirements = KubevelaAnalyzer.getRequirements(kubevela); Map nodeCounts = KubevelaAnalyzer.getNodeCount(kubevela); List controllerRequirements = getControllerRequirements(appUUID); - Main.logFile("worker-requirements-" + appUUID + ".txt", workerRequirements); - Main.logFile("worker-counts-" + appUUID + ".txt", nodeCounts); + Main.logFile("component-requirements-" + appUUID + ".txt", componentRequirements); + Main.logFile("component-counts-" + appUUID + ".txt", nodeCounts); Main.logFile("controller-requirements-" + appUUID + ".txt", controllerRequirements); // ---------------------------------------- @@ -165,8 +181,8 @@ public class NebulousAppDeployer { // Continue here while we don't really deploy // return; } - Map> workerCandidates = new HashMap<>(); - for (Map.Entry> e : workerRequirements.entrySet()) { + Map> componentCandidates = new HashMap<>(); + for (Map.Entry> e : componentRequirements.entrySet()) { String nodeName = e.getKey(); List requirements = e.getValue(); // TODO: filter by app resources (check enabled: true in resources array) @@ -177,7 +193,7 @@ public class NebulousAppDeployer { // Continue here while we don't really deploy // return; } - workerCandidates.put(nodeName, candidates); + componentCandidates.put(nodeName, candidates); } // ------------------------------------------------------------ @@ -185,14 +201,14 @@ public class NebulousAppDeployer { // Controller node log.info("Deciding on controller node candidate", keyValue("appId", appUUID), keyValue("clusterName", clusterName)); - String masterNodeName = clusterName + "-masternode"; // safe because all component node names end with a number + String masterNodeName = "N" + clusterName + "-masternode"; // safe because all component node names end with a number NodeCandidate masterNodeCandidate = null; if (controllerCandidates.size() > 0) { masterNodeCandidate = controllerCandidates.get(0); if (Set.of(NodeCandidateTypeEnum.BYON, NodeCandidateTypeEnum.EDGE) .contains(masterNodeCandidate.getNodeCandidateType())) { // Mark this candidate as already chosen - edgeCandidates.put(masterNodeName, masterNodeCandidate); + app.getNodeEdgeCandidates().put(masterNodeName, masterNodeCandidate); } } else { log.error("Empty node candidate list for controller, continuing without creating node", @@ -200,7 +216,7 @@ public class NebulousAppDeployer { } // Component nodes - log.info("Collecting worker nodes for {}", appUUID, + log.info("Collecting component nodes for {}", appUUID, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); ArrayNode nodeLabels = mapper.createArrayNode(); Map clusterNodes = new HashMap<>();; @@ -211,11 +227,11 @@ public class NebulousAppDeployer { // ExnConnector.createCluster // - Each node name and its label (nodeLabels), for // ExnConnector.labelNodes - for (Map.Entry> e : workerRequirements.entrySet()) { + for (Map.Entry> e : componentRequirements.entrySet()) { String componentName = e.getKey(); int numberOfNodes = nodeCounts.get(componentName); Set nodeNames = new HashSet<>(); - List candidates = workerCandidates.get(componentName); + List candidates = componentCandidates.get(componentName); if (candidates.size() == 0) { log.error("Empty node candidate list for component {}, continuing without creating node", componentName, keyValue("appId", appUUID), keyValue("clusterName", clusterName)); @@ -224,7 +240,7 @@ public class NebulousAppDeployer { for (int nodeNumber = 1; nodeNumber <= numberOfNodes; nodeNumber++) { String nodeName = createNodeName(clusterName, componentName, deployGeneration, nodeNumber); NodeCandidate candidate = candidates.stream() - .filter(each -> !edgeCandidates.values().contains(each)) + .filter(each -> !app.getNodeEdgeCandidates().values().contains(each)) .findFirst() .orElse(null); if (candidate == null) { @@ -233,7 +249,7 @@ public class NebulousAppDeployer { continue; } if (Set.of(NodeCandidateTypeEnum.BYON, NodeCandidateTypeEnum.EDGE).contains(candidate.getNodeCandidateType())) { - edgeCandidates.put(nodeName, candidate); + app.getNodeEdgeCandidates().put(nodeName, candidate); } clusterNodes.put(nodeName, candidate); nodeLabels.addObject().put(nodeName, "nebulouscloud.eu/" + componentName + "=true"); @@ -243,9 +259,9 @@ public class NebulousAppDeployer { } Main.logFile("nodenames-" + appUUID + ".txt", app.getComponentNodeNames()); Main.logFile("master-nodecandidate-" + appUUID + ".txt", masterNodeCandidate); - Main.logFile("worker-nodecandidates-" + appUUID + ".txt", clusterNodes); + Main.logFile("component-nodecandidates-" + appUUID + ".txt", clusterNodes); try { - Main.logFile("worker-labels-" + appUUID + ".txt", mapper.writeValueAsString(nodeLabels)); + Main.logFile("component-labels-" + appUUID + ".txt", mapper.writeValueAsString(nodeLabels)); } catch (JsonProcessingException e1) { // ignore; the labelNodes method will report the same error later } @@ -285,6 +301,23 @@ public class NebulousAppDeployer { keyValue("appId", appUUID), keyValue("clusterName", clusterName)); } + JsonNode clusterState = conn.getCluster(clusterName); + while (clusterState == null || !isClusterDeploymentFinished(clusterState)) { + // Cluster deployment includes provisioning and booting VMs, + // installing various software packages, bringing up a Kubernetes + // cluster and installing the NebulOuS runtime. This can take + // some minutes. + log.info("Waiting for cluster deployment to finish...", + keyValue("appId", appUUID), keyValue("clusterName", clusterName), + keyValue("clusterState", clusterState)); + try { + Thread.sleep(10000); + } catch (InterruptedException e1) { + // ignore + } + clusterState = conn.getCluster(clusterName); + } + log.info("Calling labelCluster", keyValue("appId", appUUID), keyValue("clusterName", clusterName)); boolean labelClusterSuccess = conn.labelNodes(appUUID, clusterName, nodeLabels); if (!labelClusterSuccess) { @@ -319,7 +352,10 @@ public class NebulousAppDeployer { // ------------------------------------------------------------ // 8. Update NebulousApp state - app.setComponentRequirements(workerRequirements); + // TODO: send out AMPL (must be done after deployCluster, once we know + // how to pass the application id into the fresh cluster) + + app.setComponentRequirements(componentRequirements); app.setComponentReplicaCounts(nodeCounts); app.setDeployedKubevela(rewritten); } @@ -372,8 +408,6 @@ public class NebulousAppDeployer { List nodesToRemove = new ArrayList<>(); ArrayNode nodesToAdd = mapper.createArrayNode(); - Map edgeCandidates = app.getNodeEdgeCandidates(); - // We know that the component names are identical and that the maps // contain all keys, so it's safe to iterate through the keys of one // map and use it in all maps. @@ -402,7 +436,7 @@ public class NebulousAppDeployer { for (int nodeNumber = 1; nodeNumber <= nAdd; nodeNumber++) { String nodeName = createNodeName(clusterName, componentName, deployGeneration, nodeNumber); NodeCandidate candidate = candidates.stream() - .filter(each -> !edgeCandidates.values().contains(each)) + .filter(each -> !app.getNodeEdgeCandidates().values().contains(each)) .findFirst() .orElse(null); if (candidate == null) { @@ -411,7 +445,7 @@ public class NebulousAppDeployer { continue; } if (Set.of(NodeCandidateTypeEnum.BYON, NodeCandidateTypeEnum.EDGE).contains(candidate.getNodeCandidateType())) { - edgeCandidates.put(nodeName, candidate); + app.getNodeEdgeCandidates().put(nodeName, candidate); } nodesToAdd.addObject() .put("nodeName", nodeName) @@ -435,7 +469,7 @@ public class NebulousAppDeployer { // nodes incur a cost. allMachineNames = app.getComponentNodeNames().get(componentName); Set removedInstances = allMachineNames.stream().limit(nRemove).collect(Collectors.toSet()); - removedInstances.forEach(edgeCandidates::remove); + removedInstances.forEach(app.getNodeEdgeCandidates()::remove); allMachineNames.removeAll(removedInstances); nodesToRemove.addAll(removedInstances); } else { @@ -458,7 +492,7 @@ public class NebulousAppDeployer { for (int nodeNumber = 1; nodeNumber <= componentReplicaCounts.get(componentName); nodeNumber++) { String nodeName = createNodeName(clusterName, componentName, deployGeneration, nodeNumber); NodeCandidate candidate = candidates.stream() - .filter(each -> !edgeCandidates.values().contains(each)) + .filter(each -> !app.getNodeEdgeCandidates().values().contains(each)) .findFirst() .orElse(null); if (candidate == null) { @@ -467,7 +501,7 @@ public class NebulousAppDeployer { continue; } if (Set.of(NodeCandidateTypeEnum.BYON, NodeCandidateTypeEnum.EDGE).contains(candidate.getNodeCandidateType())) { - edgeCandidates.put(nodeName, candidate); + app.getNodeEdgeCandidates().put(nodeName, candidate); } nodesToAdd.addObject() .put("nodeName", nodeName) @@ -489,6 +523,8 @@ public class NebulousAppDeployer { // Call `deployApplication` // Call `scaleIn` with nodesToRemove + + // Update app status } } diff --git a/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/package-info.java b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/package-info.java new file mode 100644 index 0000000..15f6162 --- /dev/null +++ b/optimiser-controller/src/main/java/eu/nebulouscloud/optimiser/controller/package-info.java @@ -0,0 +1,25 @@ +/** + * This package implements the NebulOuS optimiser controller. + * + *

The optimiser-controller keeps track of all NebulOuS applications + * running, and controls deployment and redeployment. For initial deployment, + * the optimiser-controller calculates the node requirements, negotiates node + * candidates with the resource boker and instructs the deployment manager to + * create an application cluster. For redeployment, the optimiser-controller + * listens to solutions from the per-application solver, calculates the new + * deployment scenario, and redeploys the application by instructing the + * deployment manager to add and remove nodes from the application cluster as + * needed. + * + *

The two main entry points are the classes {@link Main} (for deployment) + * and {@link LocalExecution} (for local testing). The class {@link + * NebulousApp} implements the per-applicatin state. For reasons of per-class + * code size, some methods are implemented in static classes like {@link + * AMPLGenerator} and {@link NebulousAppDeployer}. The class {@link + * ExnConnector} implements communication with the rest of the system by + * setting up the necessary publishers and message handlers to send and + * receive ActiveMQ messages. + * + * @author Rudolf Schlatte + */ +package eu.nebulouscloud.optimiser.controller; diff --git a/optimiser-controller/src/main/resources/logback.xml b/optimiser-controller/src/main/resources/logback.xml new file mode 100644 index 0000000..e0be93c --- /dev/null +++ b/optimiser-controller/src/main/resources/logback.xml @@ -0,0 +1,11 @@ + + + + level_value + + + + + + +