diff --git a/WindowsAgent/WindowsAgent.sln b/WindowsAgent/WindowsAgent.sln index 71a494b..b77d57c 100644 --- a/WindowsAgent/WindowsAgent.sln +++ b/WindowsAgent/WindowsAgent.sln @@ -3,6 +3,8 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 2012 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowsAgent", "WindowsAgent\WindowsAgent.csproj", "{F7E2A8D5-6D24-4651-A4BC-1024D59F4903}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExecutionPlanGenerator", "ExecutionPlanGenerator\ExecutionPlanGenerator.csproj", "{501BE151-4B8C-4355-88DC-3AEF1921B2D7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -13,6 +15,10 @@ Global {F7E2A8D5-6D24-4651-A4BC-1024D59F4903}.Debug|Any CPU.Build.0 = Debug|Any CPU {F7E2A8D5-6D24-4651-A4BC-1024D59F4903}.Release|Any CPU.ActiveCfg = Release|Any CPU {F7E2A8D5-6D24-4651-A4BC-1024D59F4903}.Release|Any CPU.Build.0 = Release|Any CPU + {501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/WindowsAgent/WindowsAgent/App.config b/WindowsAgent/WindowsAgent/App.config index 3df6972..6568f0f 100644 --- a/WindowsAgent/WindowsAgent/App.config +++ b/WindowsAgent/WindowsAgent/App.config @@ -1,4 +1,4 @@ - +
@@ -11,7 +11,6 @@ - @@ -20,11 +19,12 @@ - - - + + + - + + \ No newline at end of file diff --git a/WindowsAgent/WindowsAgent/MqMessage.cs b/WindowsAgent/WindowsAgent/MqMessage.cs index d77ab79..255fcb2 100644 --- a/WindowsAgent/WindowsAgent/MqMessage.cs +++ b/WindowsAgent/WindowsAgent/MqMessage.cs @@ -14,8 +14,13 @@ namespace Mirantis.Keero.WindowsAgent { this.ackFunc = ackFunc; } + + public MqMessage() + { + } public string Body { get; set; } + public string Id { get; set; } public void Ack() { diff --git a/WindowsAgent/WindowsAgent/PlanExecutor.cs b/WindowsAgent/WindowsAgent/PlanExecutor.cs index ad4910c..e1d8292 100644 --- a/WindowsAgent/WindowsAgent/PlanExecutor.cs +++ b/WindowsAgent/WindowsAgent/PlanExecutor.cs @@ -5,12 +5,15 @@ using System.Linq; using System.Management.Automation; using System.Management.Automation.Runspaces; using System.Text; +using NLog; using Newtonsoft.Json; namespace Mirantis.Keero.WindowsAgent { class PlanExecutor { + private static readonly Logger Log = LogManager.GetCurrentClassLogger(); + class ExecutionResult { public bool IsException { get; set; } @@ -26,13 +29,14 @@ namespace Mirantis.Keero.WindowsAgent public bool RebootNeeded { get; set; } - public string Execute() + public void Execute() { RebootNeeded = false; + var resultPath = this.path + ".result"; + Runspace runSpace = null; try { var plan = JsonConvert.DeserializeObject(File.ReadAllText(this.path)); - var resultPath = this.path + ".result"; List currentResults = null; try { @@ -44,7 +48,7 @@ namespace Mirantis.Keero.WindowsAgent } - var runSpace = RunspaceFactory.CreateRunspace(); + runSpace = RunspaceFactory.CreateRunspace(); runSpace.Open(); var runSpaceInvoker = new RunspaceInvoke(runSpace); @@ -70,6 +74,11 @@ namespace Mirantis.Keero.WindowsAgent psCommand.Parameters.Add(kvp.Key, kvp.Value); } } + + Log.Info("Executing {0} {1}", command.Name, string.Join(" ", + (command.Arguments ?? new Dictionary()).Select( + t => string.Format("{0}={1}", t.Key, t.Value == null ? "null" : t.Value.ToString())))); + pipeline.Commands.Add(psCommand); try { @@ -90,6 +99,7 @@ namespace Mirantis.Keero.WindowsAgent exception.GetType().FullName, exception.Message } }); + break; } finally { @@ -115,16 +125,27 @@ namespace Mirantis.Keero.WindowsAgent RebootNeeded = true; } } + File.WriteAllText(resultPath, executionResult); - File.Delete(resultPath); - return executionResult; } catch (Exception ex) { - return JsonConvert.SerializeObject(new ExecutionResult { + File.WriteAllText(resultPath, JsonConvert.SerializeObject(new ExecutionResult { IsException = true, Result = ex.Message - }, Formatting.Indented); + }, Formatting.Indented)); + } + finally + { + if (runSpace != null) + { + try + { + runSpace.Close(); + } + catch + {} + } } } diff --git a/WindowsAgent/WindowsAgent/Program.cs b/WindowsAgent/WindowsAgent/Program.cs index 32041ae..08fa125 100644 --- a/WindowsAgent/WindowsAgent/Program.cs +++ b/WindowsAgent/WindowsAgent/Program.cs @@ -1,7 +1,11 @@ using System; using System.ComponentModel; +using System.Diagnostics; using System.IO; +using System.Linq; +using System.Management.Automation; using System.Net; +using System.Text; using System.Threading; using NLog; @@ -14,6 +18,8 @@ namespace Mirantis.Keero.WindowsAgent private volatile bool stop; private Thread thread; private RabbitMqClient rabbitMqClient; + private int delayFactor = 1; + private string plansDir; static void Main(string[] args) { @@ -23,34 +29,75 @@ namespace Mirantis.Keero.WindowsAgent protected override void OnStart(string[] args) { base.OnStart(args); + + Log.Info("Version 0.3"); + this.rabbitMqClient = new RabbitMqClient(); + + var basePath = Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName); + this.plansDir = Path.Combine(basePath, "plans"); + + + if (!Directory.Exists(plansDir)) + { + Directory.CreateDirectory(plansDir); + } + this.thread = new Thread(Loop); this.thread.Start(); } void Loop() { - var doReboot = false; - const string filePath = "data.json"; + const string unknownName = "unknown"; while (!stop) { try { - if (!File.Exists(filePath)) + foreach (var file in Directory.GetFiles(this.plansDir, "*.json.result") + .Where(file => !File.Exists(Path.Combine(this.plansDir, Path.GetFileNameWithoutExtension(file))))) + { + var id = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(file)) ?? unknownName; + if (id.Equals(unknownName, StringComparison.InvariantCultureIgnoreCase)) + { + id = ""; + } + + var result = File.ReadAllText(file); + Log.Info("Sending results for {0}", id ?? unknownName); + rabbitMqClient.SendResult(new MqMessage { Body = result, Id = id }); + File.Delete(file); + } + + var path = Directory.EnumerateFiles(this.plansDir, "*.json").FirstOrDefault(); + if (path == null) { var message = rabbitMqClient.GetMessage(); - File.WriteAllText(filePath, message.Body); + var id = message.Id; + if(string.IsNullOrEmpty(id)) + { + id = unknownName; + } + + path = Path.Combine(this.plansDir, string.Format("{0}.json", id)); + File.WriteAllText(path, message.Body); + Log.Info("Received new execution plan {0}", id); message.Ack(); } - var executor = new PlanExecutor(filePath); - var result = executor.Execute(); - if(stop) break; - rabbitMqClient.SendResult(result); - File.Delete(filePath); + else + { + var id = Path.GetFileNameWithoutExtension(path); + Log.Info("Executing exising plan {0}", id); + } + var executor = new PlanExecutor(path); + executor.Execute(); + File.Delete(path); + delayFactor = 1; + + if (stop) break; if (executor.RebootNeeded) { - doReboot = true; - break; + Reboot(); } } catch (Exception exception) @@ -59,16 +106,40 @@ namespace Mirantis.Keero.WindowsAgent } } - if (doReboot) + + } + + private void Reboot() + { + Log.Info("Going for reboot!!"); + LogManager.Flush(); + /*try { - try + System.Diagnostics.Process.Start("shutdown.exe", "-r -t 0"); + } + catch (Exception ex) + { + Log.ErrorException("Cannot execute shutdown.exe", ex); + }*/ + + + try + { + PowerShell.Create().AddCommand("Restart-Computer").AddParameter("Force").Invoke(); + } + catch (Exception exception) + { + + Log.FatalException("Reboot exception", exception); + } + finally + { + Log.Info("Waiting for reboot"); + for (var i = 0; i < 10 * 60 * 5 && !stop; i++) { - System.Diagnostics.Process.Start("shutdown.exe", "-r -t 0"); - } - catch (Exception ex) - { - Log.ErrorException("Cannot execute shutdown.exe", ex); + Thread.Sleep(100); } + Log.Info("Done waiting for reboot"); } } @@ -78,18 +149,18 @@ namespace Mirantis.Keero.WindowsAgent if (stop) return; Log.WarnException("Exception in main loop", exception); var i = 0; - while (!stop && i < 10) + while (!stop && i < 10 * (delayFactor * delayFactor)) { Thread.Sleep(100); i++; } + delayFactor = Math.Min(delayFactor + 1, 6); } protected override void OnStop() { stop = true; this.rabbitMqClient.Dispose(); - Console.WriteLine("Stop"); base.OnStop(); } diff --git a/WindowsAgent/WindowsAgent/RabbitMqClient.cs b/WindowsAgent/WindowsAgent/RabbitMqClient.cs index c927bd8..6dfe5c6 100644 --- a/WindowsAgent/WindowsAgent/RabbitMqClient.cs +++ b/WindowsAgent/WindowsAgent/RabbitMqClient.cs @@ -5,12 +5,14 @@ using System.Linq; using System.Net; using System.Text; using System.Threading.Tasks; +using NLog; using RabbitMQ.Client; namespace Mirantis.Keero.WindowsAgent { class RabbitMqClient : IDisposable { + private static readonly Logger Log = LogManager.GetCurrentClassLogger(); private static readonly ConnectionFactory connectionFactory; private IConnection currentConnecton; @@ -43,21 +45,22 @@ namespace Mirantis.Keero.WindowsAgent } var session = connection.CreateModel(); session.BasicQos(0, 1, false); - session.QueueDeclare(queueName, true, false, false, null); + //session.QueueDeclare(queueName, true, false, false, null); var consumer = new QueueingBasicConsumer(session); var consumeTag = session.BasicConsume(queueName, false, consumer); - var e = (RabbitMQ.Client.Events.BasicDeliverEventArgs)consumer.Queue.Dequeue(); + var e = (RabbitMQ.Client.Events.BasicDeliverEventArgs) consumer.Queue.Dequeue(); Action ackFunc = delegate { session.BasicAck(e.DeliveryTag, false); session.BasicCancel(consumeTag); session.Close(); }; - + return new MqMessage(ackFunc) { - Body = Encoding.UTF8.GetString(e.Body) + Body = Encoding.UTF8.GetString(e.Body), + Id = e.BasicProperties.MessageId }; } - catch (Exception) + catch (Exception exception) { Dispose(); @@ -65,10 +68,11 @@ namespace Mirantis.Keero.WindowsAgent } } - public void SendResult(string text) + public void SendResult(MqMessage message) { var exchangeName = ConfigurationManager.AppSettings["rabbitmq.resultExchange"] ?? ""; - var resultQueue = ConfigurationManager.AppSettings["rabbitmq.resultQueue"] ?? "-execution-results"; + var resultRoutingKey = ConfigurationManager.AppSettings["rabbitmq.resultRoutingKey"] ?? "-execution-results"; + bool durable = bool.Parse(ConfigurationManager.AppSettings["rabbitmq.durableMessages"] ?? "true"); try { @@ -78,18 +82,19 @@ namespace Mirantis.Keero.WindowsAgent connection = this.currentConnecton = this.currentConnecton ?? connectionFactory.CreateConnection(); } var session = connection.CreateModel(); - if (!string.IsNullOrEmpty(resultQueue)) + /*if (!string.IsNullOrEmpty(resultQueue)) { - session.QueueDeclare(resultQueue, true, false, false, null); + //session.QueueDeclare(resultQueue, true, false, false, null); if (!string.IsNullOrEmpty(exchangeName)) { session.ExchangeBind(exchangeName, resultQueue, resultQueue); } - } + }*/ var basicProperties = session.CreateBasicProperties(); - basicProperties.SetPersistent(true); + basicProperties.SetPersistent(durable); + basicProperties.MessageId = message.Id; basicProperties.ContentType = "application/json"; - session.BasicPublish(exchangeName, resultQueue, basicProperties, Encoding.UTF8.GetBytes(text)); + session.BasicPublish(exchangeName, resultRoutingKey, basicProperties, Encoding.UTF8.GetBytes(message.Body)); session.Close(); } catch (Exception) diff --git a/WindowsAgent/WindowsAgent/WindowsAgent.csproj b/WindowsAgent/WindowsAgent/WindowsAgent.csproj index f467e1d..d19fa32 100644 --- a/WindowsAgent/WindowsAgent/WindowsAgent.csproj +++ b/WindowsAgent/WindowsAgent/WindowsAgent.csproj @@ -21,6 +21,7 @@ DEBUG;TRACE prompt 4 + false AnyCPU diff --git a/conductor/bin/app.py b/conductor/bin/app.py new file mode 100644 index 0000000..3c2619b --- /dev/null +++ b/conductor/bin/app.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +from conductor import app \ No newline at end of file diff --git a/conductor/conductor/__init__.py b/conductor/conductor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conductor/conductor/app.py b/conductor/conductor/app.py new file mode 100644 index 0000000..f4cf9a7 --- /dev/null +++ b/conductor/conductor/app.py @@ -0,0 +1,64 @@ +import datetime +import glob +import json +import time +import sys +import tornado.ioloop + +import rabbitmq +from workflow import Workflow +import cloud_formation +import windows_agent +from commands.dispatcher import CommandDispatcher +from config import Config +import reporting + +config = Config(sys.argv[1] if len(sys.argv) > 1 else None) + +rmqclient = rabbitmq.RabbitMqClient( + virtual_host=config.get_setting('rabbitmq', 'vhost', '/'), + login=config.get_setting('rabbitmq', 'login', 'guest'), + password=config.get_setting('rabbitmq', 'password', 'guest'), + host=config.get_setting('rabbitmq', 'host', 'localhost')) + + +def schedule(callback, *args, **kwargs): + tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 0.1, + lambda args=args, kwargs=kwargs: callback(*args, **kwargs)) + + +def task_received(task, message_id): + print 'Starting at', datetime.datetime.now() + reporter = reporting.Reporter(rmqclient, message_id, task['id']) + + command_dispatcher = CommandDispatcher(task['name'], rmqclient) + workflows = [] + for path in glob.glob("data/workflows/*.xml"): + print "loading", path + workflow = Workflow(path, task, command_dispatcher, config, reporter) + workflows.append(workflow) + + def loop(callback): + for workflow in workflows: + workflow.execute() + if not command_dispatcher.execute_pending(lambda: schedule(loop, callback)): + callback() + + def shutdown(): + command_dispatcher.close() + rmqclient.send('task-results', json.dumps(task), message_id=message_id) + print 'Finished at', datetime.datetime.now() + + loop(shutdown) + + +def message_received(body, message_id, **kwargs): + task_received(json.loads(body), message_id) + + +def start(): + rmqclient.subscribe("tasks", message_received) + +rmqclient.start(start) +tornado.ioloop.IOLoop.instance().start() + diff --git a/conductor/conductor/cloud_formation.py b/conductor/conductor/cloud_formation.py new file mode 100644 index 0000000..3ef3b41 --- /dev/null +++ b/conductor/conductor/cloud_formation.py @@ -0,0 +1,39 @@ +import base64 + +import xml_code_engine + + +def update_cf_stack(engine, context, body, template, + mappings, arguments, **kwargs): + command_dispatcher = context['/commandDispatcher'] + print "update-cf", template + + callback = lambda result: engine.evaluate_content( + body.find('success'), context) + + command_dispatcher.execute( + name='cf', template=template, mappings=mappings, + arguments=arguments, callback=callback) + + +def prepare_user_data(context, template='Default', **kwargs): + config = context['/config'] + with open('data/init.ps1') as init_script_file: + with open('data/templates/agent-config/%s.template' + % template) as template_file: + init_script = init_script_file.read() + template_data = template_file.read().replace( + '%RABBITMQ_HOST%', + config.get_setting('rabbitmq', 'host') or 'localhost') + + return init_script.replace( + '%WINDOWS_AGENT_CONFIG_BASE64%', + base64.b64encode(template_data)) + + +xml_code_engine.XmlCodeEngine.register_function( + update_cf_stack, "update-cf-stack") + +xml_code_engine.XmlCodeEngine.register_function( + prepare_user_data, "prepare_user_data") + diff --git a/conductor/conductor/commands/__init__.py b/conductor/conductor/commands/__init__.py new file mode 100644 index 0000000..551f6ea --- /dev/null +++ b/conductor/conductor/commands/__init__.py @@ -0,0 +1 @@ +import command diff --git a/conductor/conductor/commands/cloud_formation.py b/conductor/conductor/commands/cloud_formation.py new file mode 100644 index 0000000..0d12083 --- /dev/null +++ b/conductor/conductor/commands/cloud_formation.py @@ -0,0 +1,75 @@ +import json +import os +import uuid + +import conductor.helpers +from command import CommandBase +from subprocess import call + + +class HeatExecutor(CommandBase): + def __init__(self, stack): + self._pending_list = [] + self._stack = stack + + def execute(self, template, mappings, arguments, callback): + with open('data/templates/cf/%s.template' % template) as template_file: + template_data = template_file.read() + + template_data = conductor.helpers.transform_json( + json.loads(template_data), mappings) + + self._pending_list.append({ + 'template': template_data, + 'arguments': arguments, + 'callback': callback + }) + + def has_pending_commands(self): + return len(self._pending_list) > 0 + + def execute_pending(self, callback): + if not self.has_pending_commands(): + return False + + template = {} + arguments = {} + for t in self._pending_list: + template = conductor.helpers.merge_dicts( + template, t['template'], max_levels=2) + arguments = conductor.helpers.merge_dicts( + arguments, t['arguments'], max_levels=1) + + print 'Executing heat template', json.dumps(template), \ + 'with arguments', arguments, 'on stack', self._stack + + if not os.path.exists("tmp"): + os.mkdir("tmp") + file_name = "tmp/" + str(uuid.uuid4()) + print "Saving template to", file_name + with open(file_name, "w") as f: + f.write(json.dumps(template)) + + arguments_str = ';'.join(['%s=%s' % (key, value) + for (key, value) in arguments.items()]) + call([ + "./heat_run", "stack-create", + "-f" + file_name, + "-P" + arguments_str, + self._stack + ]) + + + callbacks = [] + for t in self._pending_list: + if t['callback']: + callbacks.append(t['callback']) + + self._pending_list = [] + + for cb in callbacks: + cb(True) + + callback() + + return True diff --git a/conductor/conductor/commands/command.py b/conductor/conductor/commands/command.py new file mode 100644 index 0000000..ca9d144 --- /dev/null +++ b/conductor/conductor/commands/command.py @@ -0,0 +1,12 @@ +class CommandBase(object): + def execute(self, **kwargs): + pass + + def execute_pending(self, callback): + return False + + def has_pending_commands(self): + return False + + def close(self): + pass diff --git a/conductor/conductor/commands/dispatcher.py b/conductor/conductor/commands/dispatcher.py new file mode 100644 index 0000000..b815ddb --- /dev/null +++ b/conductor/conductor/commands/dispatcher.py @@ -0,0 +1,45 @@ +import command +import cloud_formation +import windows_agent + + +class CommandDispatcher(command.CommandBase): + def __init__(self, environment_name, rmqclient): + self._command_map = { + 'cf': cloud_formation.HeatExecutor(environment_name), + 'agent': windows_agent.WindowsAgentExecutor( + environment_name, rmqclient) + } + + def execute(self, name, **kwargs): + self._command_map[name].execute(**kwargs) + + def execute_pending(self, callback): + result = 0 + count = [0] + + def on_result(): + count[0] -= 1 + if not count[0]: + callback() + + for command in self._command_map.values(): + count[0] += 1 + result += 1 + if not command.execute_pending(on_result): + count[0] -= 1 + result -= 1 + + return result > 0 + + + def has_pending_commands(self): + result = False + for command in self._command_map.values(): + result |= command.has_pending_commands() + + return result + + def close(self): + for t in self._command_map.values(): + t.close() diff --git a/conductor/conductor/commands/windows_agent.py b/conductor/conductor/commands/windows_agent.py new file mode 100644 index 0000000..c4747b6 --- /dev/null +++ b/conductor/conductor/commands/windows_agent.py @@ -0,0 +1,66 @@ +import json +import uuid + +import conductor.helpers +from command import CommandBase + + +class WindowsAgentExecutor(CommandBase): + def __init__(self, stack, rmqclient): + self._stack = stack + self._rmqclient = rmqclient + self._callback = None + self._pending_list = [] + self._current_pending_list = [] + rmqclient.subscribe('-execution-results', self._on_message) + + def execute(self, template, mappings, host, callback): + with open('data/templates/agent/%s.template' % + template) as template_file: + template_data = template_file.read() + + template_data = json.dumps(conductor.helpers.transform_json( + json.loads(template_data), mappings)) + + self._pending_list.append({ + 'id': str(uuid.uuid4()).lower(), + 'template': template_data, + 'host': ('%s-%s' % (self._stack, host)).lower().replace(' ', '-'), + 'callback': callback + }) + + def _on_message(self, body, message_id, **kwargs): + msg_id = message_id.lower() + item, index = conductor.helpers.find(lambda t: t['id'] == msg_id, + self._current_pending_list) + if item: + self._current_pending_list.pop(index) + item['callback'](json.loads(body)) + if self._callback and not self._current_pending_list: + cb = self._callback + self._callback = None + cb() + + def has_pending_commands(self): + return len(self._pending_list) > 0 + + def execute_pending(self, callback): + if not self.has_pending_commands(): + return False + + self._current_pending_list = self._pending_list + self._pending_list = [] + + self._callback = callback + + for rec in self._current_pending_list: + self._rmqclient.send( + queue=rec['host'], data=rec['template'], message_id=rec['id']) + print 'Sending RMQ message %s to %s' % ( + rec['template'], rec['host']) + + return True + + def close(self): + self._rmqclient.unsubscribe('-execution-results') + diff --git a/conductor/conductor/config.py b/conductor/conductor/config.py new file mode 100644 index 0000000..881d4ad --- /dev/null +++ b/conductor/conductor/config.py @@ -0,0 +1,19 @@ +from ConfigParser import SafeConfigParser + + +class Config(object): + CONFIG_PATH = './etc/app.config' + + def __init__(self, filename=None): + self.config = SafeConfigParser() + self.config.read(filename or self.CONFIG_PATH) + + def get_setting(self, section, name, default=None): + if not self.config.has_option(section, name): + return default + return self.config.get(section, name) + + def __getitem__(self, item): + parts = item.rsplit('.', 1) + return self.get_setting( + parts[0] if len(parts) == 2 else 'DEFAULT', parts[-1]) diff --git a/conductor/conductor/function_context.py b/conductor/conductor/function_context.py new file mode 100644 index 0000000..237f23e --- /dev/null +++ b/conductor/conductor/function_context.py @@ -0,0 +1,51 @@ +class Context(object): + def __init__(self, parent=None): + self._parent = parent + self._data = None + + def _get_data(self): + if self._data is None: + self._data = {} if self._parent is None \ + else self._parent._get_data().copy() + return self._data + + def __getitem__(self, item): + context, path = self._parseContext(item) + return context._get_data().get(path) + + def __setitem__(self, key, value): + context, path = self._parseContext(key) + context._get_data()[path] = value + + def _parseContext(self, path): + context = self + index = 0 + for c in path: + if c == ':' and context._parent is not None: + context = context._parent + elif c == '/': + while context._parent is not None: + context = context._parent + else: + break + + index += 1 + + return context, path[index:] + + def assign_from(self, context, copy=False): + self._parent = context._parent + self._data = context._data + if copy and self._data is not None: + self._data = self._data.copy() + + @property + def parent(self): + return self._parent + + def __str__(self): + if self._data is not None: + return str(self._data) + if self._parent: + return str(self._parent) + return str({}) \ No newline at end of file diff --git a/conductor/conductor/helpers.py b/conductor/conductor/helpers.py new file mode 100644 index 0000000..4128e16 --- /dev/null +++ b/conductor/conductor/helpers.py @@ -0,0 +1,48 @@ +import types + + +def transform_json(json, mappings): + if isinstance(json, types.ListType): + return [transform_json(t, mappings) for t in json] + + if isinstance(json, types.DictionaryType): + result = {} + for key, value in json.items(): + result[transform_json(key, mappings)] = \ + transform_json(value, mappings) + return result + + if isinstance(json, types.StringTypes) and json.startswith('$'): + value = mappings.get(json[1:]) + if value is not None: + return value + + return json + + +def merge_dicts(dict1, dict2, max_levels=0): + result = {} + for key, value in dict1.items(): + result[key] = value + if key in dict2: + other_value = dict2[key] + if max_levels == 1 or not isinstance( + other_value, types.DictionaryType): + result[key] = other_value + else: + result[key] = merge_dicts( + value, other_value, + 0 if max_levels == 0 else max_levels - 1) + for key, value in dict2.items(): + if key not in result: + result[key] = value + return result + +def find(f, seq): + """Return first item in sequence where f(item) == True.""" + index = 0 + for item in seq: + if f(item): + return item, index + index += 1 + return None, -1 diff --git a/conductor/conductor/rabbitmq.py b/conductor/conductor/rabbitmq.py new file mode 100644 index 0000000..d7c3351 --- /dev/null +++ b/conductor/conductor/rabbitmq.py @@ -0,0 +1,72 @@ +import uuid +import pika +from pika.adapters import TornadoConnection +import time + +try: + import tornado.ioloop + + IOLoop = tornado.ioloop.IOLoop +except ImportError: + IOLoop = None + + +class RabbitMqClient(object): + def __init__(self, host='localhost', login='guest', + password='guest', virtual_host='/'): + credentials = pika.PlainCredentials(login, password) + self._connection_parameters = pika.ConnectionParameters( + credentials=credentials, host=host, virtual_host=virtual_host) + self._subscriptions = {} + + def _create_connection(self): + self.connection = TornadoConnection( + parameters=self._connection_parameters, + on_open_callback=self._on_connected) + + def _on_connected(self, connection): + self._channel = connection.channel(self._on_channel_open) + + def _on_channel_open(self, channel): + self._channel = channel + if self._started_callback: + self._started_callback() + + def _on_queue_declared(self, frame, queue, callback, ctag): + def invoke_callback(ch, method_frame, header_frame, body): + callback(body=body, + message_id=header_frame.message_id or "") + + self._channel.basic_consume(invoke_callback, queue=queue, + no_ack=True, consumer_tag=ctag) + + def subscribe(self, queue, callback): + ctag = str(uuid.uuid4()) + self._subscriptions[queue] = ctag + + self._channel.queue_declare( + queue=queue, durable=True, + callback=lambda frame, ctag=ctag: self._on_queue_declared( + frame, queue, callback, ctag)) + + def unsubscribe(self, queue): + self._channel.basic_cancel(consumer_tag=self._subscriptions[queue]) + del self._subscriptions[queue] + + def start(self, callback=None): + if IOLoop is None: raise ImportError("Tornado not installed") + self._started_callback = callback + ioloop = IOLoop.instance() + self.timeout_id = ioloop.add_timeout(time.time() + 0.1, + self._create_connection) + + def send(self, queue, data, exchange="", message_id=""): + properties = pika.BasicProperties(message_id=message_id) + self._channel.queue_declare( + queue=queue, durable=True, + callback=lambda frame: self._channel.basic_publish( + exchange=exchange, routing_key=queue, + body=data, properties=properties)) + + + diff --git a/conductor/conductor/reporting.py b/conductor/conductor/reporting.py new file mode 100644 index 0000000..4dbef12 --- /dev/null +++ b/conductor/conductor/reporting.py @@ -0,0 +1,29 @@ +import xml_code_engine +import json + + +class Reporter(object): + def __init__(self, rmqclient, task_id, environment_id): + self._rmqclient = rmqclient + self._task_id = task_id + self._environment_id = environment_id + + def _report_func(self, id, entity, text, **kwargs): + msg = json.dumps({ + 'id': id, + 'entity': entity, + 'text': text, + 'environment_id': self._environment_id + }) + self._rmqclient.send( + queue='task-reports', data=msg, message_id=self._task_id) + +def _report_func(context, id, entity, text, **kwargs): + reporter = context['/reporter'] + return reporter._report_func(id, entity, text, **kwargs) + +xml_code_engine.XmlCodeEngine.register_function(_report_func, "report") + + + + diff --git a/conductor/conductor/windows_agent.py b/conductor/conductor/windows_agent.py new file mode 100644 index 0000000..287abb0 --- /dev/null +++ b/conductor/conductor/windows_agent.py @@ -0,0 +1,25 @@ +import xml_code_engine + + +def send_command(engine, context, body, template, host, mappings=None, + result=None, **kwargs): + if not mappings: mappings = {} + command_dispatcher = context['/commandDispatcher'] + + def callback(result_value): + print "Received result for %s: %s. Body is %s" % (template, result_value, body) + if result is not None: + context[result] = result_value['Result'] + + success_handler = body.find('success') + if success_handler is not None: + engine.evaluate_content(success_handler, context) + + command_dispatcher.execute(name='agent', + template=template, + mappings=mappings, + host=host, + callback=callback) + + +xml_code_engine.XmlCodeEngine.register_function(send_command, "send-command") \ No newline at end of file diff --git a/conductor/conductor/workflow.py b/conductor/conductor/workflow.py new file mode 100644 index 0000000..a39a7da --- /dev/null +++ b/conductor/conductor/workflow.py @@ -0,0 +1,178 @@ +import jsonpath +import types +import re + +import xml_code_engine +import function_context + +class Workflow(object): + def __init__(self, filename, data, command_dispatcher, config, reporter): + self._data = data + self._engine = xml_code_engine.XmlCodeEngine() + with open(filename) as xml: + self._engine.load(xml) + self._command_dispatcher = command_dispatcher + self._config = config + self._reporter = reporter + + def execute(self): + while True: + context = function_context.Context() + context['/dataSource'] = self._data + context['/commandDispatcher'] = self._command_dispatcher + context['/config'] = self._config + context['/reporter'] = self._reporter + if not self._engine.execute(context): + break + + @staticmethod + def _get_path(obj, path, create_non_existing=False): + # result = jsonpath.jsonpath(obj, '.'.join(path)) + # if not result or len(result) < 1: + # return None + # return result[0] + current = obj + for part in path: + if isinstance(current, types.ListType): + current = current[int(part)] + elif isinstance(current, types.DictionaryType): + if part not in current: + if create_non_existing: + current[part] = {} + else: + return None + current = current[part] + else: + raise ValueError() + + return current + + @staticmethod + def _set_path(obj, path, value): + current = Workflow._get_path(obj, path[:-1], True) + if isinstance(current, types.ListType): + current[int(path[-1])] = value + elif isinstance(current, types.DictionaryType): + current[path[-1]] = value + else: + raise ValueError() + + @staticmethod + def _get_relative_position(path, context): + position = context['__dataSource_currentPosition'] or [] + + index = 0 + for c in path: + if c == ':': + if len(position) > 0: + position = position[:-1] + elif c == '/': + position = [] + else: + break + + index += 1 + + return position, path[index:] + + @staticmethod + def _correct_position(path, context): + position, suffix = Workflow._get_relative_position(path, context) + + if not suffix: + return position + else: + return position + suffix.split('.') + + + @staticmethod + def _select_func(context, path='', source=None, **kwargs): + + if path.startswith('##'): + config = context['/config'] + return config[path[2:]] + elif path.startswith('#'): + return context[path[1:]] + + if source is not None: + return Workflow._get_path( + context[source], path.split('.')) + else: + return Workflow._get_path( + context['/dataSource'], + Workflow._correct_position(path, context)) + + + @staticmethod + def _set_func(path, context, body, engine, target=None, **kwargs): + body_data = engine.evaluate_content(body, context) + + if path.startswith('##'): + raise RuntimeError('Cannot modify config from XML-code') + elif path.startswith('#'): + context[':' + path[1:]] = body_data + return + + if target: + data = context[target] + position = path.split('.') + if Workflow._get_path(data, position) != body_data: + Workflow._set_path(data, position, body_data) + context['/hasSideEffects'] = True + else: + data = context['/dataSource'] + new_position = Workflow._correct_position(path, context) + if Workflow._get_path(data, new_position) != body_data: + Workflow._set_path(data, new_position, body_data) + context['/hasSideEffects'] = True + + @staticmethod + def _rule_func(match, context, body, engine, limit=0, name=None, **kwargs): + position = context['__dataSource_currentPosition'] or [] + + if name == 'marker': + print "!" + # data = context['__dataSource_currentObj'] + # if data is None: + # data = context['/dataSource'] + position, match = Workflow._get_relative_position(match, context) + data = Workflow._get_path(context['/dataSource'], position) + match = re.sub(r'@\.([\w.]+)', + r"Workflow._get_path(@, '\1'.split('.'))", match) + selected = jsonpath.jsonpath(data, match, 'IPATH') or [] + + index = 0 + for found_match in selected: + if 0 < int(limit) <= index: + break + index += 1 + new_position = position + found_match + context['__dataSource_currentPosition'] = new_position + context['__dataSource_currentObj'] = Workflow._get_path( + context['/dataSource'], new_position) + for element in body: + engine.evaluate(element, context) + if element.tag == 'rule' and context['/hasSideEffects']: + break + + @staticmethod + def _workflow_func(context, body, engine, **kwargs): + context['/hasSideEffects'] = False + for element in body: + engine.evaluate(element, context) + if element.tag == 'rule' and context['/hasSideEffects']: + return True + return False + + +xml_code_engine.XmlCodeEngine.register_function( + Workflow._rule_func, 'rule') + +xml_code_engine.XmlCodeEngine.register_function( + Workflow._workflow_func, 'workflow') + +xml_code_engine.XmlCodeEngine.register_function( + Workflow._set_func, 'set') + +xml_code_engine.XmlCodeEngine.register_function( + Workflow._select_func, 'select') diff --git a/conductor/conductor/xml_code_engine.py b/conductor/conductor/xml_code_engine.py new file mode 100644 index 0000000..fe676b0 --- /dev/null +++ b/conductor/conductor/xml_code_engine.py @@ -0,0 +1,134 @@ +#from lxml import etree +import xml.etree.ElementTree as etree +import types + +import function_context + + +class XmlCodeEngine(object): + _functionMap = {} + + def __init__(self): + self._document = None + + def load(self, file_obj): + self._document = etree.parse(file_obj) + + @staticmethod + def register_function(func, name): + XmlCodeEngine._functionMap[name] = func + + def _execute_function(self, name, element, parent_context): + if name == 'parameter': + return None + + if name not in self._functionMap: + raise KeyError('Unknown function %s' % name) + + definition = self._functionMap[name] + context = function_context.Context(parent_context) + args = {'engine': self, 'body': element, 'context': context} + + for key, value in element.items(): + args[key] = value + + for parameter in element.findall('parameter'): + args[parameter.get('name')] = self.evaluate_content( + parameter, context) + + return definition(**args) + + def evaluate(self, element, parent_context): + return self._execute_function(element.tag, element, parent_context) + + def evaluate_content(self, element, context): + parts = [element.text or ''] + do_strip = False + for sub_element in element: + if sub_element.tag == 'parameter': + continue + do_strip = True + parts.append(self._execute_function( + sub_element.tag, sub_element, context)) + parts.append(sub_element.tail or '') + + result = [] + + for t in parts: + if not isinstance(t, types.StringTypes): + result.append(t) + + return_value = result + if len(result) == 0: + return_value = ''.join(parts) + if do_strip: return_value = return_value.strip() + elif len(result) == 1: + return_value = result[0] + + return return_value + + def execute(self, parent_context=None): + root = self._document.getroot() + return self.evaluate(root, parent_context) + + +def _dict_func(engine, body, context, **kwargs): + result = {} + for item in body: + key = item.get('name') + value = engine.evaluate_content(item, context) + result[key] = value + return result + + +def _array_func(engine, body, context, **kwargs): + result = [] + for item in body: + result.append(engine.evaluate(item, context)) + return result + + +def _text_func(engine, body, context, **kwargs): + return str(engine.evaluate_content(body, context)) + + +def _int_func(engine, body, context, **kwargs): + return int(engine.evaluate_content(body, context)) + + +def _function_func(engine, body, context, **kwargs): + return lambda: engine.evaluate_content(body, context) + + +def _null_func(**kwargs): + return None + + +def _true_func(**kwargs): + return True + + +def _false_func(**kwargs): + return False + + +XmlCodeEngine.register_function(_dict_func, "map") +XmlCodeEngine.register_function(_array_func, "list") +XmlCodeEngine.register_function(_text_func, "text") +XmlCodeEngine.register_function(_int_func, "int") +XmlCodeEngine.register_function(_function_func, "function") +XmlCodeEngine.register_function(_null_func, "null") +XmlCodeEngine.register_function(_true_func, "true") +XmlCodeEngine.register_function(_false_func, "false") + + +def xprint(context, body, **kwargs): + print "------------------------ start ------------------------" + for arg in kwargs: + print "%s = %s" % (arg, kwargs[arg]) + print 'context = ', context + print 'body = %s (%s)' % (body, body.text) + print "------------------------- end -------------------------" + + +XmlCodeEngine.register_function(xprint, "print") diff --git a/conductor/data/init.ps1 b/conductor/data/init.ps1 new file mode 100644 index 0000000..620792c --- /dev/null +++ b/conductor/data/init.ps1 @@ -0,0 +1,14 @@ +#ps1 + +$WindowsAgentConfigBase64 = '%WINDOWS_AGENT_CONFIG_BASE64%' +$WindowsAgentConfigFile = "C:\Keero\Agent\WindowsAgent.exe.config" + +Import-Module CoreFunctions + +Stop-Service "Keero Agent" +Backup-File $WindowsAgentConfigFile +Remove-Item $WindowsAgentConfigFile -Force +ConvertFrom-Base64String -Base64String $WindowsAgentConfigBase64 -Path $WindowsAgentConfigFile +Exec sc.exe 'config','"Keero Agent"','start=','delayed-auto' +Start-Service 'Keero Agent' +Write-Log 'All done!' \ No newline at end of file diff --git a/conductor/data/templates/agent-config/Default.template b/conductor/data/templates/agent-config/Default.template new file mode 100644 index 0000000..54d9cb9 --- /dev/null +++ b/conductor/data/templates/agent-config/Default.template @@ -0,0 +1,30 @@ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/conductor/data/templates/agent/AskDnsIp.template b/conductor/data/templates/agent/AskDnsIp.template new file mode 100644 index 0000000..ee2057c --- /dev/null +++ b/conductor/data/templates/agent/AskDnsIp.template @@ -0,0 +1,12 @@ +{ + "Scripts": [ + "ZnVuY3Rpb24gR2V0LURuc0xpc3RlbmluZ0lwQWRkcmVzc2VzIHsNCiAgICAoR2V0LUROU1NlcnZlciAtQ29tcHV0ZXJOYW1lIGxvY2FsaG9zdCkuU2VydmVyU2V0dGluZy5MaXN0ZW5pbmdJcEFkZHJlc3MNCn0NCg==" + ], + "Commands": [ + { + "Name": "Get-DnsListeningIpAddress", + "Arguments": {} + } + ], + "RebootOnCompletion": 0 +} \ No newline at end of file diff --git a/conductor/data/templates/agent/CreatePrimaryDC.template b/conductor/data/templates/agent/CreatePrimaryDC.template new file mode 100644 index 0000000..b181dba --- /dev/null +++ b/conductor/data/templates/agent/CreatePrimaryDC.template @@ -0,0 +1,21 @@ +{ + "Scripts": [ + "RnVuY3Rpb24gU2V0LUxvY2FsVXNlclBhc3N3b3JkIHsNCiAgICBwYXJhbSAoDQogICAgICAgIFtTdHJpbmddICRVc2VyTmFtZSwNCiAgICAgICAgW1N0cmluZ10gJFBhc3N3b3JkLA0KICAgICAgICBbU3dpdGNoXSAkRm9yY2UNCiAgICApDQogICAgDQogICAgdHJhcCB7IFN0b3AtRXhlY3V0aW9uICRfIH0NCiAgICANCiAgICBpZiAoKEdldC1XbWlPYmplY3QgV2luMzJfVXNlckFjY291bnQgLUZpbHRlciAiTG9jYWxBY2NvdW50ID0gJ1RydWUnIEFORCBOYW1lPSckVXNlck5hbWUnIikgLWVxICRudWxsKSB7DQogICAgICAgIHRocm93ICJVbmFibGUgdG8gZmluZCBsb2NhbCB1c2VyIGFjY291bnQgJyRVc2VyTmFtZSciDQogICAgfQ0KICAgIA0KICAgIGlmICgkRm9yY2UpIHsNCiAgICAgICAgV3JpdGUtTG9nICJDaGFuZ2luZyBwYXNzd29yZCBmb3IgdXNlciAnJFVzZXJOYW1lJyB0byAnKioqKionIiAjIDopDQogICAgICAgIChbQURTSV0gIldpbk5UOi8vLi8kVXNlck5hbWUiKS5TZXRQYXNzd29yZCgkUGFzc3dvcmQpDQogICAgfQ0KICAgIGVsc2Ugew0KICAgICAgICBXcml0ZS1Mb2dXYXJuaW5nICJZb3UgYXJlIHRyeWluZyB0byBjaGFuZ2UgcGFzc3dvcmQgZm9yIHVzZXIgJyRVc2VyTmFtZScuIFRvIGRvIHRoaXMgcGxlYXNlIHJ1biB0aGUgY29tbWFuZCBhZ2FpbiB3aXRoIC1Gb3JjZSBwYXJhbWV0ZXIuIg0KICAgICAgICAkVXNlckFjY291bnQNCiAgICB9DQp9DQoNCg0KDQpGdW5jdGlvbiBJbnN0YWxsLVJvbGVQcmltYXJ5RG9tYWluQ29udHJvbGxlcg0Kew0KPCMNCi5TWU5PUFNJUw0KQ29uZmlndXJlIG5vZGUncyBuZXR3b3JrIGFkYXB0ZXJzLg0KQ3JlYXRlIGZpcnN0IGRvbWFpbiBjb250cm9sbGVyIGluIHRoZSBmb3Jlc3QuDQoNCi5FWEFNUExFDQpQUz4gSW5zdGFsbC1Sb2xlUHJpbWFyeURvbWFpbkNvbnRyb2xsZXIgLURvbWFpbk5hbWUgYWNtZS5sb2NhbCAtU2FmZU1vZGVQYXNzd29yZCAiUEBzc3cwcmQiDQoNCkluc3RhbGwgRE5TIGFuZCBBRERTLCBjcmVhdGUgZm9yZXN0IGFuZCBkb21haW4gJ2FjbWUubG9jYWwnLg0KU2V0IERDIHJlY292ZXJ5IG1vZGUgcGFzc3dvcmQgdG8gJ1BAc3N3MHJkJy4NCiM+DQoJDQoJcGFyYW0NCgkoDQoJCVtTdHJpbmddDQoJCSMgTmV3IGRvbWFpbiBuYW1lLg0KCQkkRG9tYWluTmFtZSwNCgkJDQoJCVtTdHJpbmddDQoJCSMgRG9tYWluIGNvbnRyb2xsZXIgcmVjb3ZlcnkgbW9kZSBwYXNzd29yZC4NCgkJJFNhZmVNb2RlUGFzc3dvcmQNCgkpDQoNCgl0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQ0KDQogICAgICAgICMgQWRkIHJlcXVpcmVkIHdpbmRvd3MgZmVhdHVyZXMNCglBZGQtV2luZG93c0ZlYXR1cmVXcmFwcGVyIGANCgkJLU5hbWUgIkROUyIsIkFELURvbWFpbi1TZXJ2aWNlcyIsIlJTQVQtREZTLU1nbXQtQ29uIiBgDQoJCS1JbmNsdWRlTWFuYWdlbWVudFRvb2xzIGANCiAgICAgICAgLU5vdGlmeVJlc3RhcnQNCg0KDQoJV3JpdGUtTG9nICJDcmVhdGluZyBmaXJzdCBkb21haW4gY29udHJvbGxlciAuLi4iDQoJCQ0KCSRTTUFQID0gQ29udmVydFRvLVNlY3VyZVN0cmluZyAtU3RyaW5nICRTYWZlTW9kZVBhc3N3b3JkIC1Bc1BsYWluVGV4dCAtRm9yY2UNCgkJDQoJSW5zdGFsbC1BRERTRm9yZXN0IGANCgkJLURvbWFpbk5hbWUgJERvbWFpbk5hbWUgYA0KCQktU2FmZU1vZGVBZG1pbmlzdHJhdG9yUGFzc3dvcmQgJFNNQVAgYA0KCQktRG9tYWluTW9kZSBEZWZhdWx0IGANCgkJLUZvcmVzdE1vZGUgRGVmYXVsdCBgDQoJCS1Ob1JlYm9vdE9uQ29tcGxldGlvbiBgDQoJCS1Gb3JjZSBgDQoJCS1FcnJvckFjdGlvbiBTdG9wIHwgT3V0LU51bGwNCg0KCVdyaXRlLUhvc3QgIldhaXRpbmcgZm9yIHJlYm9vdCAuLi4iCQkNCiMJU3RvcC1FeGVjdXRpb24gLUV4aXRDb2RlIDMwMTAgLUV4aXRTdHJpbmcgIkNvbXB1dGVyIG11c3QgYmUgcmVzdGFydGVkIHRvIGZpbmlzaCBkb21haW4gY29udHJvbGxlciBwcm9tb3Rpb24uIg0KIwlXcml0ZS1Mb2cgIlJlc3RhcmluZyBjb21wdXRlciAuLi4iDQojCVJlc3RhcnQtQ29tcHV0ZXIgLUZvcmNlDQp9DQo=" + ], + "Commands": [ + { + "Name": "Import-Module", + "Arguments": { + "Name": "CoreFunctions" + } + }, + { + "Name": "Install-RolePrimaryDomainController", + "Arguments": { + "DomainName": "$domain", + "SafeModePassword": "$recoveryPassword" + } + } + ], + "RebootOnCompletion": 1 +} \ No newline at end of file diff --git a/conductor/data/templates/agent/CreateSecondaryDC.template b/conductor/data/templates/agent/CreateSecondaryDC.template new file mode 100644 index 0000000..a5ad7f4 --- /dev/null +++ b/conductor/data/templates/agent/CreateSecondaryDC.template @@ -0,0 +1,23 @@ +{ + "Scripts": [ + "RnVuY3Rpb24gSW5zdGFsbC1Sb2xlU2Vjb25kYXJ5RG9tYWluQ29udHJvbGxlcg0Kew0KPCMNCi5TWU5PUFNJUw0KSW5zdGFsbCBhZGRpdGlvbmFsIChzZWNvbmRhcnkpIGRvbWFpbiBjb250cm9sbGVyLg0KDQojPg0KCXBhcmFtDQoJKA0KCQlbU3RyaW5nXQ0KCQkjIERvbWFpbiBuYW1lIHRvIGpvaW4gdG8uDQoJCSREb21haW5OYW1lLA0KCQkNCgkJW1N0cmluZ10NCgkJIyBEb21haW4gdXNlciB3aG8gaXMgYWxsb3dlZCB0byBqb2luIGNvbXB1dGVyIHRvIGRvbWFpbi4NCgkJJFVzZXJOYW1lLA0KCQkNCgkJW1N0cmluZ10NCgkJIyBVc2VyJ3MgcGFzc3dvcmQuDQoJCSRQYXNzd29yZCwNCgkJDQoJCVtTdHJpbmddDQoJCSMgRG9tYWluIGNvbnRyb2xsZXIgcmVjb3ZlcnkgbW9kZSBwYXNzd29yZC4NCgkJJFNhZmVNb2RlUGFzc3dvcmQNCgkpDQoNCgl0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQ0KCQ0KCSRDcmVkZW50aWFsID0gTmV3LUNyZWRlbnRpYWwgLVVzZXJOYW1lICIkRG9tYWluTmFtZVwkVXNlck5hbWUiIC1QYXNzd29yZCAkUGFzc3dvcmQNCgkJDQoJIyBBZGQgcmVxdWlyZWQgd2luZG93cyBmZWF0dXJlcw0KCUFkZC1XaW5kb3dzRmVhdHVyZVdyYXBwZXIgYA0KCQktTmFtZSAiRE5TIiwiQUQtRG9tYWluLVNlcnZpY2VzIiwiUlNBVC1ERlMtTWdtdC1Db24iIGANCgkJLUluY2x1ZGVNYW5hZ2VtZW50VG9vbHMgYA0KICAgICAgICAgICAgICAgIC1Ob3RpZnlSZXN0YXJ0DQoJCQ0KCQ0KICAgICAgICBXcml0ZS1Mb2cgIkFkZGluZyBzZWNvbmRhcnkgZG9tYWluIGNvbnRyb2xsZXIgLi4uIg0KICAgIA0KCSRTTUFQID0gQ29udmVydFRvLVNlY3VyZVN0cmluZyAtU3RyaW5nICRTYWZlTW9kZVBhc3N3b3JkIC1Bc1BsYWluVGV4dCAtRm9yY2UNCg0KCUluc3RhbGwtQUREU0RvbWFpbkNvbnRyb2xsZXIgYA0KCQktRG9tYWluTmFtZSAkRG9tYWluTmFtZSBgDQoJCS1TYWZlTW9kZUFkbWluaXN0cmF0b3JQYXNzd29yZCAkU01BUCBgDQoJCS1DcmVkZW50aWFsICRDcmVkZW50aWFsIGANCgkJLU5vUmVib290T25Db21wbGV0aW9uIGANCgkJLUZvcmNlIGANCgkJLUVycm9yQWN0aW9uIFN0b3AgfCBPdXQtTnVsbA0KDQoJV3JpdGUtTG9nICJXYWl0aW5nIGZvciByZXN0YXJ0IC4uLiINCiMJU3RvcC1FeGVjdXRpb24gLUV4aXRDb2RlIDMwMTAgLUV4aXRTdHJpbmcgIkNvbXB1dGVyIG11c3QgYmUgcmVzdGFydGVkIHRvIGZpbmlzaCBkb21haW4gY29udHJvbGxlciBwcm9tb3Rpb24uIg0KIwlXcml0ZS1Mb2cgIlJlc3RhcnRpbmcgY29tcHV0ZXIgLi4uIg0KIwlSZXN0YXJ0LUNvbXB1dGVyIC1Gb3JjZQ0KfQ0K" + ], + "Commands": [ + { + "Name": "Import-Module", + "Arguments": { + "Name": "CoreFunctions" + } + }, + { + "Name": "Install-RoleSecondaryDomainController", + "Arguments": { + "DomainName": "$domain", + "UserName": "Administrator", + "Password": "$domainPassword", + "SafeModePassword": "$recoveryPassword" + } + } + ], + "RebootOnCompletion": 1 +} \ No newline at end of file diff --git a/conductor/data/templates/agent/InstallIIS.template b/conductor/data/templates/agent/InstallIIS.template new file mode 100644 index 0000000..baeaec1 --- /dev/null +++ b/conductor/data/templates/agent/InstallIIS.template @@ -0,0 +1,12 @@ +{ + "Scripts": [ + "ZnVuY3Rpb24gSW5zdGFsbC1XZWJTZXJ2ZXIgew0KICAgIEltcG9ydC1Nb2R1bGUgU2VydmVyTWFuYWdlcg0KICAgIEluc3RhbGwtV2luZG93c0ZlYXR1cmUgV2ViLVNlcnZlciAtSW5jbHVkZU1hbmFnZW1lbnRUb29scw0KfQ0K" + ], + "Commands": [ + { + "Name": "Install-WebServer", + "Arguments": {} + } + ], + "RebootOnCompletion": 0 +} \ No newline at end of file diff --git a/conductor/data/templates/agent/JoinDomain.template b/conductor/data/templates/agent/JoinDomain.template new file mode 100644 index 0000000..85e4b9c --- /dev/null +++ b/conductor/data/templates/agent/JoinDomain.template @@ -0,0 +1,27 @@ +{ + "Scripts": [], + "Commands": [ + { + "Name": "Import-Module", + "Arguments": { + "Name": "CoreFunctions" + } + }, + { + "Name": "Set-NetworkAdapterConfiguration", + "Arguments": { + "FirstAvailable": true, + "DNSServer": "$dnsIp" + } + }, + { + "Name": "Join-Domain", + "Arguments": { + "DomainName": "$domain", + "Username": "Administrator", + "Password": "$domainPassword" + } + } + ], + "RebootOnCompletion": 1 +} \ No newline at end of file diff --git a/conductor/data/templates/agent/SetPassword.template b/conductor/data/templates/agent/SetPassword.template new file mode 100644 index 0000000..1cc9dcc --- /dev/null +++ b/conductor/data/templates/agent/SetPassword.template @@ -0,0 +1,22 @@ +{ + "Scripts": [ + "RnVuY3Rpb24gU2V0LUxvY2FsVXNlclBhc3N3b3JkIHsNCiAgICBwYXJhbSAoDQogICAgICAgIFtTdHJpbmddICRVc2VyTmFtZSwNCiAgICAgICAgW1N0cmluZ10gJFBhc3N3b3JkLA0KICAgICAgICBbU3dpdGNoXSAkRm9yY2UNCiAgICApDQogICAgDQogICAgdHJhcCB7IFN0b3AtRXhlY3V0aW9uICRfIH0NCiAgICANCiAgICBpZiAoKEdldC1XbWlPYmplY3QgV2luMzJfVXNlckFjY291bnQgLUZpbHRlciAiTG9jYWxBY2NvdW50ID0gJ1RydWUnIEFORCBOYW1lPSckVXNlck5hbWUnIikgLWVxICRudWxsKSB7DQogICAgICAgIHRocm93ICJVbmFibGUgdG8gZmluZCBsb2NhbCB1c2VyIGFjY291bnQgJyRVc2VyTmFtZSciDQogICAgfQ0KICAgIA0KICAgIGlmICgkRm9yY2UpIHsNCiAgICAgICAgV3JpdGUtTG9nICJDaGFuZ2luZyBwYXNzd29yZCBmb3IgdXNlciAnJFVzZXJOYW1lJyB0byAnKioqKionIiAjIDopDQogICAgICAgIChbQURTSV0gIldpbk5UOi8vLi8kVXNlck5hbWUiKS5TZXRQYXNzd29yZCgkUGFzc3dvcmQpDQogICAgfQ0KICAgIGVsc2Ugew0KICAgICAgICBXcml0ZS1Mb2dXYXJuaW5nICJZb3UgYXJlIHRyeWluZyB0byBjaGFuZ2UgcGFzc3dvcmQgZm9yIHVzZXIgJyRVc2VyTmFtZScuIFRvIGRvIHRoaXMgcGxlYXNlIHJ1biB0aGUgY29tbWFuZCBhZ2FpbiB3aXRoIC1Gb3JjZSBwYXJhbWV0ZXIuIg0KICAgICAgICAkVXNlckFjY291bnQNCiAgICB9DQp9DQoNCg0KDQpGdW5jdGlvbiBJbnN0YWxsLVJvbGVQcmltYXJ5RG9tYWluQ29udHJvbGxlcg0Kew0KPCMNCi5TWU5PUFNJUw0KQ29uZmlndXJlIG5vZGUncyBuZXR3b3JrIGFkYXB0ZXJzLg0KQ3JlYXRlIGZpcnN0IGRvbWFpbiBjb250cm9sbGVyIGluIHRoZSBmb3Jlc3QuDQoNCi5FWEFNUExFDQpQUz4gSW5zdGFsbC1Sb2xlUHJpbWFyeURvbWFpbkNvbnRyb2xsZXIgLURvbWFpbk5hbWUgYWNtZS5sb2NhbCAtU2FmZU1vZGVQYXNzd29yZCAiUEBzc3cwcmQiDQoNCkluc3RhbGwgRE5TIGFuZCBBRERTLCBjcmVhdGUgZm9yZXN0IGFuZCBkb21haW4gJ2FjbWUubG9jYWwnLg0KU2V0IERDIHJlY292ZXJ5IG1vZGUgcGFzc3dvcmQgdG8gJ1BAc3N3MHJkJy4NCiM+DQoJDQoJcGFyYW0NCgkoDQoJCVtTdHJpbmddDQoJCSMgTmV3IGRvbWFpbiBuYW1lLg0KCQkkRG9tYWluTmFtZSwNCgkJDQoJCVtTdHJpbmddDQoJCSMgRG9tYWluIGNvbnRyb2xsZXIgcmVjb3ZlcnkgbW9kZSBwYXNzd29yZC4NCgkJJFNhZmVNb2RlUGFzc3dvcmQNCgkpDQoNCgl0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQ0KDQogICAgICAgICMgQWRkIHJlcXVpcmVkIHdpbmRvd3MgZmVhdHVyZXMNCglBZGQtV2luZG93c0ZlYXR1cmVXcmFwcGVyIGANCgkJLU5hbWUgIkROUyIsIkFELURvbWFpbi1TZXJ2aWNlcyIsIlJTQVQtREZTLU1nbXQtQ29uIiBgDQoJCS1JbmNsdWRlTWFuYWdlbWVudFRvb2xzIGANCiAgICAgICAgLU5vdGlmeVJlc3RhcnQNCg0KDQoJV3JpdGUtTG9nICJDcmVhdGluZyBmaXJzdCBkb21haW4gY29udHJvbGxlciAuLi4iDQoJCQ0KCSRTTUFQID0gQ29udmVydFRvLVNlY3VyZVN0cmluZyAtU3RyaW5nICRTYWZlTW9kZVBhc3N3b3JkIC1Bc1BsYWluVGV4dCAtRm9yY2UNCgkJDQoJSW5zdGFsbC1BRERTRm9yZXN0IGANCgkJLURvbWFpbk5hbWUgJERvbWFpbk5hbWUgYA0KCQktU2FmZU1vZGVBZG1pbmlzdHJhdG9yUGFzc3dvcmQgJFNNQVAgYA0KCQktRG9tYWluTW9kZSBEZWZhdWx0IGANCgkJLUZvcmVzdE1vZGUgRGVmYXVsdCBgDQoJCS1Ob1JlYm9vdE9uQ29tcGxldGlvbiBgDQoJCS1Gb3JjZSBgDQoJCS1FcnJvckFjdGlvbiBTdG9wIHwgT3V0LU51bGwNCg0KCVdyaXRlLUhvc3QgIldhaXRpbmcgZm9yIHJlYm9vdCAuLi4iCQkNCiMJU3RvcC1FeGVjdXRpb24gLUV4aXRDb2RlIDMwMTAgLUV4aXRTdHJpbmcgIkNvbXB1dGVyIG11c3QgYmUgcmVzdGFydGVkIHRvIGZpbmlzaCBkb21haW4gY29udHJvbGxlciBwcm9tb3Rpb24uIg0KIwlXcml0ZS1Mb2cgIlJlc3RhcmluZyBjb21wdXRlciAuLi4iDQojCVJlc3RhcnQtQ29tcHV0ZXIgLUZvcmNlDQp9DQo=" + ], + "Commands": [ + { + "Name": "Import-Module", + "Arguments": { + "Name": "CoreFunctions" + } + }, + { + "Name": "Set-LocalUserPassword", + "Arguments": { + "UserName": "Administrator", + "Password": "$adminPassword", + "Force": true + } + } + ], + "RebootOnCompletion": 0 +} \ No newline at end of file diff --git a/conductor/data/templates/cf/Windows.template b/conductor/data/templates/cf/Windows.template new file mode 100644 index 0000000..e14069f --- /dev/null +++ b/conductor/data/templates/cf/Windows.template @@ -0,0 +1,62 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + + "Description" : "", + + "Parameters" : { + "KeyName" : { + "Description" : "Name of an existing Amazon EC2 key pair for RDP access", + "Type" : "String", + "Default" : "keero_key" + }, + "InstanceType" : { + "Description" : "Amazon EC2 instance type", + "Type" : "String", + "Default" : "m1.medium", + "AllowedValues" : [ "m1.small", "m1.medium", "m1.large" ] + }, + "ImageName" : { + "Description" : "Image name", + "Type" : "String", + "Default" : "ws-2012-full-agent", + "AllowedValues" : [ "ws-2012-full", "ws-2012-core", "ws-2012-full-agent" ] + } + }, + + "Resources" : { + "IAMUser" : { + "Type" : "AWS::IAM::User", + "Properties" : { + "Path": "/", + "Policies": [{ + "PolicyName": "root", + "PolicyDocument": { "Statement":[{ + "Effect": "Allow", + "Action": "CloudFormation:DescribeStackResource", + "Resource": "*" + }]} + }] + } + }, + + "IAMUserAccessKey" : { + "Type" : "AWS::IAM::AccessKey", + "Properties" : { + "UserName" : {"Ref": "IAMUser"} + } + }, + + "$instanceName": { + "Type" : "AWS::EC2::Instance", + "Properties": { + "InstanceType" : { "Ref" : "InstanceType" }, + "ImageId" : { "Ref" : "ImageName" }, + "KeyName" : { "Ref" : "KeyName" }, + "UserData": "$userData" + } + } + }, + + "Outputs" : { + } +} \ No newline at end of file diff --git a/conductor/data/workflows/AD.xml b/conductor/data/workflows/AD.xml new file mode 100644 index 0000000..751d477 --- /dev/null +++ b/conductor/data/workflows/AD.xml @@ -0,0 +1,199 @@ + + + + + Creating instance + + + + + + + + + keero-linux-keys + m1.medium + ws-2012-full-agent + + + + + + Instance + + + + + + + + + + + + + + + + + + + + + Creating Primary Domain Controller on unit + + + + + + + + + + + Primary Domain Controller created + + + + + + + + + + + + + + + + + + + + ' and @.state.primaryDcIp)] + + + + + + + + + + + + + + + Unit + + + + + + + + + + + + + + + + + + Secondary Domain Controller created + + + created + + + + + \ No newline at end of file diff --git a/conductor/data/workflows/IIS.xml b/conductor/data/workflows/IIS.xml new file mode 100644 index 0000000..40bf0ab --- /dev/null +++ b/conductor/data/workflows/IIS.xml @@ -0,0 +1,60 @@ + + + + + Creating instance + + + + + + + + + keero-linux-keys + m1.medium + ws-2012-full-agent + + + + + + Instance + Creating IIS Web Server on unit + + + + + has started + + + + + \ No newline at end of file diff --git a/conductor/etc/app.config b/conductor/etc/app.config new file mode 100644 index 0000000..f69fe45 --- /dev/null +++ b/conductor/etc/app.config @@ -0,0 +1,5 @@ +[rabbitmq] +host = localhost +vhost = keero +login = keero +password = keero \ No newline at end of file diff --git a/conductor/test.json b/conductor/test.json new file mode 100644 index 0000000..2427070 --- /dev/null +++ b/conductor/test.json @@ -0,0 +1,41 @@ +{ + "name": "MyDataCenter", + "id": "adc6d143f9584d10808c7ef4d07e4802", + "services": { + "activeDirectories": [ + { + "id": "9571747991184642B95F430A014616F9", + "domain": "acme.loc", + "adminPassword": "SuperP@ssw0rd!", + "units": [ + { + "id": "273c9183b6e74c9c9db7fdd532c5eb25", + "name": "dc01", + "isMaster": true, + "recoveryPassword": "2SuperP@ssw0rd2" + }, + { + "id": "377c6f16d17a416791f80724dab360c6", + "name": "dc02", + "isMaster": false, + "adminPassword": "SuperP@ssw0rd", + "recoveryPassword": "2SuperP@ssw0rd2" + } + ] + } + ], + "webServers": [ + { + "id": "e9657ceef84a4e669e31795040080262", + "domain": "acme.loc", + "units": [ + { + "id": "e6f9cfd07ced48fba64e6bd9e65aba64", + "name": "iis01", + "adminPassword": "SuperP@ssw0rd" + } + ] + } + ] + } +} diff --git a/conductor/tools/install_venv.py b/conductor/tools/install_venv.py new file mode 100644 index 0000000..c3b8171 --- /dev/null +++ b/conductor/tools/install_venv.py @@ -0,0 +1,154 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Installation script for Glance's development virtualenv +""" + +import os +import subprocess +import sys + + +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +VENV = os.path.join(ROOT, '.venv') +PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') +TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires') + + +def die(message, *args): + print >> sys.stderr, message % args + sys.exit(1) + + +def run_command(cmd, redirect_output=True, check_exit_code=True): + """ + Runs a command in an out-of-process shell, returning the + output of that command. Working directory is ROOT. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return output + + +HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'], + check_exit_code=False).strip()) +HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'], + check_exit_code=False).strip()) + + +def check_dependencies(): + """Make sure virtualenv is in the path.""" + + if not HAS_VIRTUALENV: + print 'not found.' + # Try installing it via easy_install... + if HAS_EASY_INSTALL: + print 'Installing virtualenv via easy_install...', + if not run_command(['which', 'easy_install']): + die('ERROR: virtualenv not found.\n\n' + 'Balancer development requires virtualenv, please install' + ' it using your favorite package management tool') + print 'done.' + print 'done.' + + +def create_virtualenv(venv=VENV): + """ + Creates the virtual environment and installs PIP only into the + virtual environment + """ + print 'Creating venv...', + run_command(['virtualenv', '-q', '--no-site-packages', VENV]) + print 'done.' + print 'Installing pip in virtualenv...', + if not run_command(['tools/with_venv.sh', 'easy_install', + 'pip>1.0']).strip(): + die("Failed to install pip.") + print 'done.' + + +def pip_install(*args): + run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + +def install_dependencies(venv=VENV): + print 'Installing dependencies with pip (this can take a while)...' + + pip_install('pip') + + pip_install('-r', PIP_REQUIRES) + pip_install('-r', TEST_REQUIRES) + + # Tell the virtual env how to "import glance" + py_ver = _detect_python_version(venv) + pthfile = os.path.join(venv, "lib", py_ver, + "site-packages", "balancer.pth") + f = open(pthfile, 'w') + f.write("%s\n" % ROOT) + + +def _detect_python_version(venv): + lib_dir = os.path.join(venv, "lib") + for pathname in os.listdir(lib_dir): + if pathname.startswith('python'): + return pathname + raise Exception('Unable to detect Python version') + + +def print_help(): + help = """ + Glance development environment setup is complete. + + Glance development uses virtualenv to track and manage Python dependencies + while in development and testing. + + To activate the Glance virtualenv for the extent of your current shell session + you can run: + + $ source .venv/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print help + + +def main(argv): + check_dependencies() + create_virtualenv() + install_dependencies() + print_help() + +if __name__ == '__main__': + main(sys.argv) diff --git a/conductor/tools/pip-requires b/conductor/tools/pip-requires new file mode 100644 index 0000000..a7bcbfe --- /dev/null +++ b/conductor/tools/pip-requires @@ -0,0 +1,3 @@ +pika +tornado +jsonpath \ No newline at end of file diff --git a/conductor/tools/test-requires b/conductor/tools/test-requires new file mode 100644 index 0000000..d69cfd4 --- /dev/null +++ b/conductor/tools/test-requires @@ -0,0 +1,8 @@ +unittest2 +mock==0.8.0 +nose +nose-exclude +nosexcover +#openstack.nose_plugin +pep8==1.0.1 +sphinx>=1.1.2 diff --git a/conductor/tools/with_venv.sh b/conductor/tools/with_venv.sh new file mode 100644 index 0000000..ae91bbc --- /dev/null +++ b/conductor/tools/with_venv.sh @@ -0,0 +1,4 @@ +#!/bin/bash +TOOLS=`dirname $0` +VENV=$TOOLS/../.venv +source $VENV/bin/activate && $@