stackforge-attic/murano-docs
Code Issues Proposed changes

Merge branch 'feature-orchestration'

Browse Source
This commit is contained in:
Timur Nurlygayanov 2013-03-11 20:51:34 +04:00
commit 7b3bab4854
41 changed files with 1712 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
WindowsAgent/WindowsAgent.sln
View File

@ -3,6 +3,8 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowsAgent", "WindowsAgent\WindowsAgent.csproj", "{F7E2A8D5-6D24-4651-A4BC-1024D59F4903}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExecutionPlanGenerator", "ExecutionPlanGenerator\ExecutionPlanGenerator.csproj", "{501BE151-4B8C-4355-88DC-3AEF1921B2D7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -13,6 +15,10 @@ Global
{F7E2A8D5-6D24-4651-A4BC-1024D59F4903}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F7E2A8D5-6D24-4651-A4BC-1024D59F4903}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F7E2A8D5-6D24-4651-A4BC-1024D59F4903}.Release|Any CPU.Build.0 = Release|Any CPU
{501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{501BE151-4B8C-4355-88DC-3AEF1921B2D7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

12
WindowsAgent/WindowsAgent/App.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8" ?>
<?xml version="1.0" encoding="UTF-8" ?>
<configuration>
<configSections>
<section name="nlog" type="NLog.Config.ConfigSectionHandler, NLog"/>
@ -11,7 +11,6 @@
<targets>
<target name="file" xsi:type="File" fileName="${basedir}/log.txt"
layout="${date} ${level}: &lt;${logger:shortName=true}&gt; ${message} ${exception:format=tostring}"/>
</targets>
<rules>
@ -20,11 +19,12 @@
</nlog>
<appSettings>
<add key="rabbitmq.host" value="localhost"/>
<add key="rabbitmq.user" value="guest"/>
<add key="rabbitmq.password" value="guest"/>
<add key="rabbitmq.vhost" value="/"/>
<add key="rabbitmq.user" value="keero"/>
<add key="rabbitmq.password" value="keero"/>
<add key="rabbitmq.vhost" value="keero"/>
<add key="rabbitmq.resultExchange" value=""/>
<add key="rabbitmq.resultQueue" value="-execution-results"/>
<add key="rabbitmq.resultRoutingKey" value="-execution-results"/>
<add key="rabbitmq.durableMessages" value="true"/>
</appSettings>
</configuration>

5
WindowsAgent/WindowsAgent/MqMessage.cs
View File

@ -14,8 +14,13 @@ namespace Mirantis.Keero.WindowsAgent
{
this.ackFunc = ackFunc;
}
public MqMessage()
{
}
public string Body { get; set; }
public string Id { get; set; }
public void Ack()
{

35
WindowsAgent/WindowsAgent/PlanExecutor.cs
View File

@ -5,12 +5,15 @@ using System.Linq;
using System.Management.Automation;
using System.Management.Automation.Runspaces;
using System.Text;
using NLog;
using Newtonsoft.Json;
namespace Mirantis.Keero.WindowsAgent
{
class PlanExecutor
{
private static readonly Logger Log = LogManager.GetCurrentClassLogger();
class ExecutionResult
{
public bool IsException { get; set; }
@ -26,13 +29,14 @@ namespace Mirantis.Keero.WindowsAgent
public bool RebootNeeded { get; set; }
public string Execute()
public void Execute()
{
RebootNeeded = false;
var resultPath = this.path + ".result";
Runspace runSpace = null;
try
{
var plan = JsonConvert.DeserializeObject<ExecutionPlan>(File.ReadAllText(this.path));
var resultPath = this.path + ".result";
List<ExecutionResult> currentResults = null;
try
{
@ -44,7 +48,7 @@ namespace Mirantis.Keero.WindowsAgent
}
var runSpace = RunspaceFactory.CreateRunspace();
runSpace = RunspaceFactory.CreateRunspace();
runSpace.Open();
var runSpaceInvoker = new RunspaceInvoke(runSpace);
@ -70,6 +74,11 @@ namespace Mirantis.Keero.WindowsAgent
psCommand.Parameters.Add(kvp.Key, kvp.Value);
}
}
Log.Info("Executing {0} {1}", command.Name, string.Join(" ",
(command.Arguments ?? new Dictionary<string, object>()).Select(
t => string.Format("{0}={1}", t.Key, t.Value == null ? "null" : t.Value.ToString()))));
pipeline.Commands.Add(psCommand);
try
{
@ -90,6 +99,7 @@ namespace Mirantis.Keero.WindowsAgent
exception.GetType().FullName, exception.Message
}
});
break;
}
finally
{
@ -115,16 +125,27 @@ namespace Mirantis.Keero.WindowsAgent
RebootNeeded = true;
}
}
File.WriteAllText(resultPath, executionResult);
File.Delete(resultPath);
return executionResult;
}
catch (Exception ex)
{
return JsonConvert.SerializeObject(new ExecutionResult {
File.WriteAllText(resultPath, JsonConvert.SerializeObject(new ExecutionResult {
IsException = true,
Result = ex.Message
}, Formatting.Indented);
}, Formatting.Indented));
}
finally
{
if (runSpace != null)
{
try
{
runSpace.Close();
}
catch
{}
}
}
}

111
WindowsAgent/WindowsAgent/Program.cs
View File

@ -1,7 +1,11 @@
using System;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Management.Automation;
using System.Net;
using System.Text;
using System.Threading;
using NLog;
@ -14,6 +18,8 @@ namespace Mirantis.Keero.WindowsAgent
private volatile bool stop;
private Thread thread;
private RabbitMqClient rabbitMqClient;
private int delayFactor = 1;
private string plansDir;
static void Main(string[] args)
{
@ -23,34 +29,75 @@ namespace Mirantis.Keero.WindowsAgent
protected override void OnStart(string[] args)
{
base.OnStart(args);
Log.Info("Version 0.3");
this.rabbitMqClient = new RabbitMqClient();
var basePath = Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName);
this.plansDir = Path.Combine(basePath, "plans");
if (!Directory.Exists(plansDir))
{
Directory.CreateDirectory(plansDir);
}
this.thread = new Thread(Loop);
this.thread.Start();
}
void Loop()
{
var doReboot = false;
const string filePath = "data.json";
const string unknownName = "unknown";
while (!stop)
{
try
{
if (!File.Exists(filePath))
foreach (var file in Directory.GetFiles(this.plansDir, "*.json.result")
.Where(file => !File.Exists(Path.Combine(this.plansDir, Path.GetFileNameWithoutExtension(file)))))
{
var id = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(file)) ?? unknownName;
if (id.Equals(unknownName, StringComparison.InvariantCultureIgnoreCase))
{
id = "";
}
var result = File.ReadAllText(file);
Log.Info("Sending results for {0}", id ?? unknownName);
rabbitMqClient.SendResult(new MqMessage { Body = result, Id = id });
File.Delete(file);
}
var path = Directory.EnumerateFiles(this.plansDir, "*.json").FirstOrDefault();
if (path == null)
{
var message = rabbitMqClient.GetMessage();
File.WriteAllText(filePath, message.Body);
var id = message.Id;
if(string.IsNullOrEmpty(id))
{
id = unknownName;
}
path = Path.Combine(this.plansDir, string.Format("{0}.json", id));
File.WriteAllText(path, message.Body);
Log.Info("Received new execution plan {0}", id);
message.Ack();
}
var executor = new PlanExecutor(filePath);
var result = executor.Execute();
if(stop) break;
rabbitMqClient.SendResult(result);
File.Delete(filePath);
else
{
var id = Path.GetFileNameWithoutExtension(path);
Log.Info("Executing exising plan {0}", id);
}
var executor = new PlanExecutor(path);
executor.Execute();
File.Delete(path);
delayFactor = 1;
if (stop) break;
if (executor.RebootNeeded)
{
doReboot = true;
break;
Reboot();
}
}
catch (Exception exception)
@ -59,16 +106,40 @@ namespace Mirantis.Keero.WindowsAgent
}
}
if (doReboot)
}
private void Reboot()
{
Log.Info("Going for reboot!!");
LogManager.Flush();
/*try
{
try
System.Diagnostics.Process.Start("shutdown.exe", "-r -t 0");
}
catch (Exception ex)
{
Log.ErrorException("Cannot execute shutdown.exe", ex);
}*/
try
{
PowerShell.Create().AddCommand("Restart-Computer").AddParameter("Force").Invoke();
}
catch (Exception exception)
{
Log.FatalException("Reboot exception", exception);
}
finally
{
Log.Info("Waiting for reboot");
for (var i = 0; i < 10 * 60 * 5 && !stop; i++)
{
System.Diagnostics.Process.Start("shutdown.exe", "-r -t 0");
}
catch (Exception ex)
{
Log.ErrorException("Cannot execute shutdown.exe", ex);
Thread.Sleep(100);
}
Log.Info("Done waiting for reboot");
}
}
@ -78,18 +149,18 @@ namespace Mirantis.Keero.WindowsAgent
if (stop) return;
Log.WarnException("Exception in main loop", exception);
var i = 0;
while (!stop && i < 10)
while (!stop && i < 10 * (delayFactor * delayFactor))
{
Thread.Sleep(100);
i++;
}
delayFactor = Math.Min(delayFactor + 1, 6);
}
protected override void OnStop()
{
stop = true;
this.rabbitMqClient.Dispose();
Console.WriteLine("Stop");
base.OnStop();
}

29
WindowsAgent/WindowsAgent/RabbitMqClient.cs
View File

@ -5,12 +5,14 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using NLog;
using RabbitMQ.Client;
namespace Mirantis.Keero.WindowsAgent
{
class RabbitMqClient : IDisposable
{
private static readonly Logger Log = LogManager.GetCurrentClassLogger();
private static readonly ConnectionFactory connectionFactory;
private IConnection currentConnecton;
@ -43,21 +45,22 @@ namespace Mirantis.Keero.WindowsAgent
}
var session = connection.CreateModel();
session.BasicQos(0, 1, false);
session.QueueDeclare(queueName, true, false, false, null);
//session.QueueDeclare(queueName, true, false, false, null);
var consumer = new QueueingBasicConsumer(session);
var consumeTag = session.BasicConsume(queueName, false, consumer);
var e = (RabbitMQ.Client.Events.BasicDeliverEventArgs)consumer.Queue.Dequeue();
var e = (RabbitMQ.Client.Events.BasicDeliverEventArgs) consumer.Queue.Dequeue();
Action ackFunc = delegate {
session.BasicAck(e.DeliveryTag, false);
session.BasicCancel(consumeTag);
session.Close();
};
return new MqMessage(ackFunc) {
Body = Encoding.UTF8.GetString(e.Body)
Body = Encoding.UTF8.GetString(e.Body),
Id = e.BasicProperties.MessageId
};
}
catch (Exception)
catch (Exception exception)
{
Dispose();
@ -65,10 +68,11 @@ namespace Mirantis.Keero.WindowsAgent
}
}
public void SendResult(string text)
public void SendResult(MqMessage message)
{
var exchangeName = ConfigurationManager.AppSettings["rabbitmq.resultExchange"] ?? "";
var resultQueue = ConfigurationManager.AppSettings["rabbitmq.resultQueue"] ?? "-execution-results";
var resultRoutingKey = ConfigurationManager.AppSettings["rabbitmq.resultRoutingKey"] ?? "-execution-results";
bool durable = bool.Parse(ConfigurationManager.AppSettings["rabbitmq.durableMessages"] ?? "true");
try
{
@ -78,18 +82,19 @@ namespace Mirantis.Keero.WindowsAgent
connection = this.currentConnecton = this.currentConnecton ?? connectionFactory.CreateConnection();
}
var session = connection.CreateModel();
if (!string.IsNullOrEmpty(resultQueue))
/*if (!string.IsNullOrEmpty(resultQueue))
{
session.QueueDeclare(resultQueue, true, false, false, null);
//session.QueueDeclare(resultQueue, true, false, false, null);
if (!string.IsNullOrEmpty(exchangeName))
{
session.ExchangeBind(exchangeName, resultQueue, resultQueue);
}
}
}*/
var basicProperties = session.CreateBasicProperties();
basicProperties.SetPersistent(true);
basicProperties.SetPersistent(durable);
basicProperties.MessageId = message.Id;
basicProperties.ContentType = "application/json";
session.BasicPublish(exchangeName, resultQueue, basicProperties, Encoding.UTF8.GetBytes(text));
session.BasicPublish(exchangeName, resultRoutingKey, basicProperties, Encoding.UTF8.GetBytes(message.Body));
session.Close();
}
catch (Exception)

1
WindowsAgent/WindowsAgent/WindowsAgent.csproj
View File

@ -21,6 +21,7 @@
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>

3
conductor/bin/app.py Normal file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env python
from conductor import app

0
conductor/conductor/__init__.py Normal file
View File

64
conductor/conductor/app.py Normal file
View File

@ -0,0 +1,64 @@
import datetime
import glob
import json
import time
import sys
import tornado.ioloop
import rabbitmq
from workflow import Workflow
import cloud_formation
import windows_agent
from commands.dispatcher import CommandDispatcher
from config import Config
import reporting
config = Config(sys.argv[1] if len(sys.argv) > 1 else None)
rmqclient = rabbitmq.RabbitMqClient(
virtual_host=config.get_setting('rabbitmq', 'vhost', '/'),
login=config.get_setting('rabbitmq', 'login', 'guest'),
password=config.get_setting('rabbitmq', 'password', 'guest'),
host=config.get_setting('rabbitmq', 'host', 'localhost'))
def schedule(callback, *args, **kwargs):
tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 0.1,
lambda args=args, kwargs=kwargs: callback(*args, **kwargs))
def task_received(task, message_id):
print 'Starting at', datetime.datetime.now()
reporter = reporting.Reporter(rmqclient, message_id, task['id'])
command_dispatcher = CommandDispatcher(task['name'], rmqclient)
workflows = []
for path in glob.glob("data/workflows/*.xml"):
print "loading", path
workflow = Workflow(path, task, command_dispatcher, config, reporter)
workflows.append(workflow)
def loop(callback):
for workflow in workflows:
workflow.execute()
if not command_dispatcher.execute_pending(lambda: schedule(loop, callback)):
callback()
def shutdown():
command_dispatcher.close()
rmqclient.send('task-results', json.dumps(task), message_id=message_id)
print 'Finished at', datetime.datetime.now()
loop(shutdown)
def message_received(body, message_id, **kwargs):
task_received(json.loads(body), message_id)
def start():
rmqclient.subscribe("tasks", message_received)
rmqclient.start(start)
tornado.ioloop.IOLoop.instance().start()

39
conductor/conductor/cloud_formation.py Normal file
View File

@ -0,0 +1,39 @@
import base64
import xml_code_engine
def update_cf_stack(engine, context, body, template,
mappings, arguments, **kwargs):
command_dispatcher = context['/commandDispatcher']
print "update-cf", template
callback = lambda result: engine.evaluate_content(
body.find('success'), context)
command_dispatcher.execute(
name='cf', template=template, mappings=mappings,
arguments=arguments, callback=callback)
def prepare_user_data(context, template='Default', **kwargs):
config = context['/config']
with open('data/init.ps1') as init_script_file:
with open('data/templates/agent-config/%s.template'
% template) as template_file:
init_script = init_script_file.read()
template_data = template_file.read().replace(
'%RABBITMQ_HOST%',
config.get_setting('rabbitmq', 'host') or 'localhost')
return init_script.replace(
'%WINDOWS_AGENT_CONFIG_BASE64%',
base64.b64encode(template_data))
xml_code_engine.XmlCodeEngine.register_function(
update_cf_stack, "update-cf-stack")
xml_code_engine.XmlCodeEngine.register_function(
prepare_user_data, "prepare_user_data")

1
conductor/conductor/commands/__init__.py Normal file
View File

@ -0,0 +1 @@
import command

75
conductor/conductor/commands/cloud_formation.py Normal file
View File

@ -0,0 +1,75 @@
import json
import os
import uuid
import conductor.helpers
from command import CommandBase
from subprocess import call
class HeatExecutor(CommandBase):
def __init__(self, stack):
self._pending_list = []
self._stack = stack
def execute(self, template, mappings, arguments, callback):
with open('data/templates/cf/%s.template' % template) as template_file:
template_data = template_file.read()
template_data = conductor.helpers.transform_json(
json.loads(template_data), mappings)
self._pending_list.append({
'template': template_data,
'arguments': arguments,
'callback': callback
})
def has_pending_commands(self):
return len(self._pending_list) > 0
def execute_pending(self, callback):
if not self.has_pending_commands():
return False
template = {}
arguments = {}
for t in self._pending_list:
template = conductor.helpers.merge_dicts(
template, t['template'], max_levels=2)
arguments = conductor.helpers.merge_dicts(
arguments, t['arguments'], max_levels=1)
print 'Executing heat template', json.dumps(template), \
'with arguments', arguments, 'on stack', self._stack
if not os.path.exists("tmp"):
os.mkdir("tmp")
file_name = "tmp/" + str(uuid.uuid4())
print "Saving template to", file_name
with open(file_name, "w") as f:
f.write(json.dumps(template))
arguments_str = ';'.join(['%s=%s' % (key, value)
for (key, value) in arguments.items()])
call([
"./heat_run", "stack-create",
"-f" + file_name,
"-P" + arguments_str,
self._stack
])
callbacks = []
for t in self._pending_list:
if t['callback']:
callbacks.append(t['callback'])
self._pending_list = []
for cb in callbacks:
cb(True)
callback()
return True

12
conductor/conductor/commands/command.py Normal file
View File

@ -0,0 +1,12 @@
class CommandBase(object):
def execute(self, **kwargs):
pass
def execute_pending(self, callback):
return False
def has_pending_commands(self):
return False
def close(self):
pass

45
conductor/conductor/commands/dispatcher.py Normal file
View File

@ -0,0 +1,45 @@
import command
import cloud_formation
import windows_agent
class CommandDispatcher(command.CommandBase):
def __init__(self, environment_name, rmqclient):
self._command_map = {
'cf': cloud_formation.HeatExecutor(environment_name),
'agent': windows_agent.WindowsAgentExecutor(
environment_name, rmqclient)
}
def execute(self, name, **kwargs):
self._command_map[name].execute(**kwargs)
def execute_pending(self, callback):
result = 0
count = [0]
def on_result():
count[0] -= 1
if not count[0]:
callback()
for command in self._command_map.values():
count[0] += 1
result += 1
if not command.execute_pending(on_result):
count[0] -= 1
result -= 1
return result > 0
def has_pending_commands(self):
result = False
for command in self._command_map.values():
result |= command.has_pending_commands()
return result
def close(self):
for t in self._command_map.values():
t.close()

66
conductor/conductor/commands/windows_agent.py Normal file
View File

@ -0,0 +1,66 @@
import json
import uuid
import conductor.helpers
from command import CommandBase
class WindowsAgentExecutor(CommandBase):
def __init__(self, stack, rmqclient):
self._stack = stack
self._rmqclient = rmqclient
self._callback = None
self._pending_list = []
self._current_pending_list = []
rmqclient.subscribe('-execution-results', self._on_message)
def execute(self, template, mappings, host, callback):
with open('data/templates/agent/%s.template' %
template) as template_file:
template_data = template_file.read()
template_data = json.dumps(conductor.helpers.transform_json(
json.loads(template_data), mappings))
self._pending_list.append({
'id': str(uuid.uuid4()).lower(),
'template': template_data,
'host': ('%s-%s' % (self._stack, host)).lower().replace(' ', '-'),
'callback': callback
})
def _on_message(self, body, message_id, **kwargs):
msg_id = message_id.lower()
item, index = conductor.helpers.find(lambda t: t['id'] == msg_id,
self._current_pending_list)
if item:
self._current_pending_list.pop(index)
item['callback'](json.loads(body))
if self._callback and not self._current_pending_list:
cb = self._callback
self._callback = None
cb()
def has_pending_commands(self):
return len(self._pending_list) > 0
def execute_pending(self, callback):
if not self.has_pending_commands():
return False
self._current_pending_list = self._pending_list
self._pending_list = []
self._callback = callback
for rec in self._current_pending_list:
self._rmqclient.send(
queue=rec['host'], data=rec['template'], message_id=rec['id'])
print 'Sending RMQ message %s to %s' % (
rec['template'], rec['host'])
return True
def close(self):
self._rmqclient.unsubscribe('-execution-results')

19
conductor/conductor/config.py Normal file
View File

@ -0,0 +1,19 @@
from ConfigParser import SafeConfigParser
class Config(object):
CONFIG_PATH = './etc/app.config'
def __init__(self, filename=None):
self.config = SafeConfigParser()
self.config.read(filename or self.CONFIG_PATH)
def get_setting(self, section, name, default=None):
if not self.config.has_option(section, name):
return default
return self.config.get(section, name)
def __getitem__(self, item):
parts = item.rsplit('.', 1)
return self.get_setting(
parts[0] if len(parts) == 2 else 'DEFAULT', parts[-1])

51
conductor/conductor/function_context.py Normal file
View File

@ -0,0 +1,51 @@
class Context(object):
def __init__(self, parent=None):
self._parent = parent
self._data = None
def _get_data(self):
if self._data is None:
self._data = {} if self._parent is None \
else self._parent._get_data().copy()
return self._data
def __getitem__(self, item):
context, path = self._parseContext(item)
return context._get_data().get(path)
def __setitem__(self, key, value):
context, path = self._parseContext(key)
context._get_data()[path] = value
def _parseContext(self, path):
context = self
index = 0
for c in path:
if c == ':' and context._parent is not None:
context = context._parent
elif c == '/':
while context._parent is not None:
context = context._parent
else:
break
index += 1
return context, path[index:]
def assign_from(self, context, copy=False):
self._parent = context._parent
self._data = context._data
if copy and self._data is not None:
self._data = self._data.copy()
@property
def parent(self):
return self._parent
def __str__(self):
if self._data is not None:
return str(self._data)
if self._parent:
return str(self._parent)
return str({})

48
conductor/conductor/helpers.py Normal file
View File

@ -0,0 +1,48 @@
import types
def transform_json(json, mappings):
if isinstance(json, types.ListType):
return [transform_json(t, mappings) for t in json]
if isinstance(json, types.DictionaryType):
result = {}
for key, value in json.items():
result[transform_json(key, mappings)] = \
transform_json(value, mappings)
return result
if isinstance(json, types.StringTypes) and json.startswith('$'):
value = mappings.get(json[1:])
if value is not None:
return value
return json
def merge_dicts(dict1, dict2, max_levels=0):
result = {}
for key, value in dict1.items():
result[key] = value
if key in dict2:
other_value = dict2[key]
if max_levels == 1 or not isinstance(
other_value, types.DictionaryType):
result[key] = other_value
else:
result[key] = merge_dicts(
value, other_value,
0 if max_levels == 0 else max_levels - 1)
for key, value in dict2.items():
if key not in result:
result[key] = value
return result
def find(f, seq):
"""Return first item in sequence where f(item) == True."""
index = 0
for item in seq:
if f(item):
return item, index
index += 1
return None, -1

72
conductor/conductor/rabbitmq.py Normal file
View File

@ -0,0 +1,72 @@
import uuid
import pika
from pika.adapters import TornadoConnection
import time
try:
import tornado.ioloop
IOLoop = tornado.ioloop.IOLoop
except ImportError:
IOLoop = None
class RabbitMqClient(object):
def __init__(self, host='localhost', login='guest',
password='guest', virtual_host='/'):
credentials = pika.PlainCredentials(login, password)
self._connection_parameters = pika.ConnectionParameters(
credentials=credentials, host=host, virtual_host=virtual_host)
self._subscriptions = {}
def _create_connection(self):
self.connection = TornadoConnection(
parameters=self._connection_parameters,
on_open_callback=self._on_connected)
def _on_connected(self, connection):
self._channel = connection.channel(self._on_channel_open)
def _on_channel_open(self, channel):
self._channel = channel
if self._started_callback:
self._started_callback()
def _on_queue_declared(self, frame, queue, callback, ctag):
def invoke_callback(ch, method_frame, header_frame, body):
callback(body=body,
message_id=header_frame.message_id or "")
self._channel.basic_consume(invoke_callback, queue=queue,
no_ack=True, consumer_tag=ctag)
def subscribe(self, queue, callback):
ctag = str(uuid.uuid4())
self._subscriptions[queue] = ctag
self._channel.queue_declare(
queue=queue, durable=True,
callback=lambda frame, ctag=ctag: self._on_queue_declared(
frame, queue, callback, ctag))
def unsubscribe(self, queue):
self._channel.basic_cancel(consumer_tag=self._subscriptions[queue])
del self._subscriptions[queue]
def start(self, callback=None):
if IOLoop is None: raise ImportError("Tornado not installed")
self._started_callback = callback
ioloop = IOLoop.instance()
self.timeout_id = ioloop.add_timeout(time.time() + 0.1,
self._create_connection)
def send(self, queue, data, exchange="", message_id=""):
properties = pika.BasicProperties(message_id=message_id)
self._channel.queue_declare(
queue=queue, durable=True,
callback=lambda frame: self._channel.basic_publish(
exchange=exchange, routing_key=queue,
body=data, properties=properties))

29
conductor/conductor/reporting.py Normal file
View File

@ -0,0 +1,29 @@
import xml_code_engine
import json
class Reporter(object):
def __init__(self, rmqclient, task_id, environment_id):
self._rmqclient = rmqclient
self._task_id = task_id
self._environment_id = environment_id
def _report_func(self, id, entity, text, **kwargs):
msg = json.dumps({
'id': id,
'entity': entity,
'text': text,
'environment_id': self._environment_id
})
self._rmqclient.send(
queue='task-reports', data=msg, message_id=self._task_id)
def _report_func(context, id, entity, text, **kwargs):
reporter = context['/reporter']
return reporter._report_func(id, entity, text, **kwargs)
xml_code_engine.XmlCodeEngine.register_function(_report_func, "report")

25
conductor/conductor/windows_agent.py Normal file
View File

@ -0,0 +1,25 @@
import xml_code_engine
def send_command(engine, context, body, template, host, mappings=None,
result=None, **kwargs):
if not mappings: mappings = {}
command_dispatcher = context['/commandDispatcher']
def callback(result_value):
print "Received result for %s: %s. Body is %s" % (template, result_value, body)
if result is not None:
context[result] = result_value['Result']
success_handler = body.find('success')
if success_handler is not None:
engine.evaluate_content(success_handler, context)
command_dispatcher.execute(name='agent',
template=template,
mappings=mappings,
host=host,
callback=callback)
xml_code_engine.XmlCodeEngine.register_function(send_command, "send-command")

178
conductor/conductor/workflow.py Normal file
View File

@ -0,0 +1,178 @@
import jsonpath
import types
import re
import xml_code_engine
import function_context
class Workflow(object):
def __init__(self, filename, data, command_dispatcher, config, reporter):
self._data = data
self._engine = xml_code_engine.XmlCodeEngine()
with open(filename) as xml:
self._engine.load(xml)
self._command_dispatcher = command_dispatcher
self._config = config
self._reporter = reporter
def execute(self):
while True:
context = function_context.Context()
context['/dataSource'] = self._data
context['/commandDispatcher'] = self._command_dispatcher
context['/config'] = self._config
context['/reporter'] = self._reporter
if not self._engine.execute(context):
break
@staticmethod
def _get_path(obj, path, create_non_existing=False):
# result = jsonpath.jsonpath(obj, '.'.join(path))
# if not result or len(result) < 1:
# return None
# return result[0]
current = obj
for part in path:
if isinstance(current, types.ListType):
current = current[int(part)]
elif isinstance(current, types.DictionaryType):
if part not in current:
if create_non_existing:
current[part] = {}
else:
return None
current = current[part]
else:
raise ValueError()
return current
@staticmethod
def _set_path(obj, path, value):
current = Workflow._get_path(obj, path[:-1], True)
if isinstance(current, types.ListType):
current[int(path[-1])] = value
elif isinstance(current, types.DictionaryType):
current[path[-1]] = value
else:
raise ValueError()
@staticmethod
def _get_relative_position(path, context):
position = context['__dataSource_currentPosition'] or []
index = 0
for c in path:
if c == ':':
if len(position) > 0:
position = position[:-1]
elif c == '/':
position = []
else:
break
index += 1
return position, path[index:]
@staticmethod
def _correct_position(path, context):
position, suffix = Workflow._get_relative_position(path, context)
if not suffix:
return position
else:
return position + suffix.split('.')
@staticmethod
def _select_func(context, path='', source=None, **kwargs):
if path.startswith('##'):
config = context['/config']
return config[path[2:]]
elif path.startswith('#'):
return context[path[1:]]
if source is not None:
return Workflow._get_path(
context[source], path.split('.'))
else:
return Workflow._get_path(
context['/dataSource'],
Workflow._correct_position(path, context))
@staticmethod
def _set_func(path, context, body, engine, target=None, **kwargs):
body_data = engine.evaluate_content(body, context)
if path.startswith('##'):
raise RuntimeError('Cannot modify config from XML-code')
elif path.startswith('#'):
context[':' + path[1:]] = body_data
return
if target:
data = context[target]
position = path.split('.')
if Workflow._get_path(data, position) != body_data:
Workflow._set_path(data, position, body_data)
context['/hasSideEffects'] = True
else:
data = context['/dataSource']
new_position = Workflow._correct_position(path, context)
if Workflow._get_path(data, new_position) != body_data:
Workflow._set_path(data, new_position, body_data)
context['/hasSideEffects'] = True
@staticmethod
def _rule_func(match, context, body, engine, limit=0, name=None, **kwargs):
position = context['__dataSource_currentPosition'] or []
if name == 'marker':
print "!"
# data = context['__dataSource_currentObj']
# if data is None:
# data = context['/dataSource']
position, match = Workflow._get_relative_position(match, context)
data = Workflow._get_path(context['/dataSource'], position)
match = re.sub(r'@\.([\w.]+)',
r"Workflow._get_path(@, '\1'.split('.'))", match)
selected = jsonpath.jsonpath(data, match, 'IPATH') or []
index = 0
for found_match in selected:
if 0 < int(limit) <= index:
break
index += 1
new_position = position + found_match
context['__dataSource_currentPosition'] = new_position
context['__dataSource_currentObj'] = Workflow._get_path(
context['/dataSource'], new_position)
for element in body:
engine.evaluate(element, context)
if element.tag == 'rule' and context['/hasSideEffects']:
break
@staticmethod
def _workflow_func(context, body, engine, **kwargs):
context['/hasSideEffects'] = False
for element in body:
engine.evaluate(element, context)
if element.tag == 'rule' and context['/hasSideEffects']:
return True
return False
xml_code_engine.XmlCodeEngine.register_function(
Workflow._rule_func, 'rule')
xml_code_engine.XmlCodeEngine.register_function(
Workflow._workflow_func, 'workflow')
xml_code_engine.XmlCodeEngine.register_function(
Workflow._set_func, 'set')
xml_code_engine.XmlCodeEngine.register_function(
Workflow._select_func, 'select')

134
conductor/conductor/xml_code_engine.py Normal file
View File

@ -0,0 +1,134 @@
#from lxml import etree
import xml.etree.ElementTree as etree
import types
import function_context
class XmlCodeEngine(object):
_functionMap = {}
def __init__(self):
self._document = None
def load(self, file_obj):
self._document = etree.parse(file_obj)
@staticmethod
def register_function(func, name):
XmlCodeEngine._functionMap[name] = func
def _execute_function(self, name, element, parent_context):
if name == 'parameter':
return None
if name not in self._functionMap:
raise KeyError('Unknown function %s' % name)
definition = self._functionMap[name]
context = function_context.Context(parent_context)
args = {'engine': self, 'body': element, 'context': context}
for key, value in element.items():
args[key] = value
for parameter in element.findall('parameter'):
args[parameter.get('name')] = self.evaluate_content(
parameter, context)
return definition(**args)
def evaluate(self, element, parent_context):
return self._execute_function(element.tag, element, parent_context)
def evaluate_content(self, element, context):
parts = [element.text or '']
do_strip = False
for sub_element in element:
if sub_element.tag == 'parameter':
continue
do_strip = True
parts.append(self._execute_function(
sub_element.tag, sub_element, context))
parts.append(sub_element.tail or '')
result = []
for t in parts:
if not isinstance(t, types.StringTypes):
result.append(t)
return_value = result
if len(result) == 0:
return_value = ''.join(parts)
if do_strip: return_value = return_value.strip()
elif len(result) == 1:
return_value = result[0]
return return_value
def execute(self, parent_context=None):
root = self._document.getroot()
return self.evaluate(root, parent_context)
def _dict_func(engine, body, context, **kwargs):
result = {}
for item in body:
key = item.get('name')
value = engine.evaluate_content(item, context)
result[key] = value
return result
def _array_func(engine, body, context, **kwargs):
result = []
for item in body:
result.append(engine.evaluate(item, context))
return result
def _text_func(engine, body, context, **kwargs):
return str(engine.evaluate_content(body, context))
def _int_func(engine, body, context, **kwargs):
return int(engine.evaluate_content(body, context))
def _function_func(engine, body, context, **kwargs):
return lambda: engine.evaluate_content(body, context)
def _null_func(**kwargs):
return None
def _true_func(**kwargs):
return True
def _false_func(**kwargs):
return False
XmlCodeEngine.register_function(_dict_func, "map")
XmlCodeEngine.register_function(_array_func, "list")
XmlCodeEngine.register_function(_text_func, "text")
XmlCodeEngine.register_function(_int_func, "int")
XmlCodeEngine.register_function(_function_func, "function")
XmlCodeEngine.register_function(_null_func, "null")
XmlCodeEngine.register_function(_true_func, "true")
XmlCodeEngine.register_function(_false_func, "false")
def xprint(context, body, **kwargs):
print "------------------------ start ------------------------"
for arg in kwargs:
print "%s = %s" % (arg, kwargs[arg])
print 'context = ', context
print 'body = %s (%s)' % (body, body.text)
print "------------------------- end -------------------------"
XmlCodeEngine.register_function(xprint, "print")

14
conductor/data/init.ps1 Normal file
View File

@ -0,0 +1,14 @@
#ps1
$WindowsAgentConfigBase64 = '%WINDOWS_AGENT_CONFIG_BASE64%'
$WindowsAgentConfigFile = "C:\Keero\Agent\WindowsAgent.exe.config"
Import-Module CoreFunctions
Stop-Service "Keero Agent"
Backup-File $WindowsAgentConfigFile
Remove-Item $WindowsAgentConfigFile -Force
ConvertFrom-Base64String -Base64String $WindowsAgentConfigBase64 -Path $WindowsAgentConfigFile
Exec sc.exe 'config','"Keero Agent"','start=','delayed-auto'
Start-Service 'Keero Agent'
Write-Log 'All done!'

30
conductor/data/templates/agent-config/Default.template Normal file
View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8" ?>
<configuration>
<configSections>
<section name="nlog" type="NLog.Config.ConfigSectionHandler, NLog"/>
</configSections>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<targets>
<target name="file" xsi:type="File" fileName="${basedir}/log.txt"
layout="${date} ${level}: &lt;${logger:shortName=true}&gt; ${message} ${exception:format=tostring}"/>
</targets>
<rules>
<logger name="*" minlevel="Debug" writeTo="file" />
</rules>
</nlog>
<appSettings>
<add key="rabbitmq.host" value="%RABBITMQ_HOST%"/>
<add key="rabbitmq.user" value="keero"/>
<add key="rabbitmq.password" value="keero"/>
<add key="rabbitmq.vhost" value="keero"/>
<add key="rabbitmq.resultExchange" value=""/>
<add key="rabbitmq.resultRoutingKey" value="-execution-results"/>
<add key="rabbitmq.durableMessages" value="true"/>
</appSettings>
</configuration>

12
conductor/data/templates/agent/AskDnsIp.template Normal file
View File

@ -0,0 +1,12 @@
{
"Scripts": [
"ZnVuY3Rpb24gR2V0LURuc0xpc3RlbmluZ0lwQWRkcmVzc2VzIHsNCiAgICAoR2V0LUROU1NlcnZlciAtQ29tcHV0ZXJOYW1lIGxvY2FsaG9zdCkuU2VydmVyU2V0dGluZy5MaXN0ZW5pbmdJcEFkZHJlc3MNCn0NCg=="
],
"Commands": [
{
"Name": "Get-DnsListeningIpAddress",
"Arguments": {}
}
],
"RebootOnCompletion": 0
}

21
conductor/data/templates/agent/CreatePrimaryDC.template Normal file
View File

@ -0,0 +1,21 @@
{
"Scripts": [
"RnVuY3Rpb24gU2V0LUxvY2FsVXNlclBhc3N3b3JkIHsNCiAgICBwYXJhbSAoDQogICAgICAgIFtTdHJpbmddICRVc2VyTmFtZSwNCiAgICAgICAgW1N0cmluZ10gJFBhc3N3b3JkLA0KICAgICAgICBbU3dpdGNoXSAkRm9yY2UNCiAgICApDQogICAgDQogICAgdHJhcCB7IFN0b3AtRXhlY3V0aW9uICRfIH0NCiAgICANCiAgICBpZiAoKEdldC1XbWlPYmplY3QgV2luMzJfVXNlckFjY291bnQgLUZpbHRlciAiTG9jYWxBY2NvdW50ID0gJ1RydWUnIEFORCBOYW1lPSckVXNlck5hbWUnIikgLWVxICRudWxsKSB7DQogICAgICAgIHRocm93ICJVbmFibGUgdG8gZmluZCBsb2NhbCB1c2VyIGFjY291bnQgJyRVc2VyTmFtZSciDQogICAgfQ0KICAgIA0KICAgIGlmICgkRm9yY2UpIHsNCiAgICAgICAgV3JpdGUtTG9nICJDaGFuZ2luZyBwYXNzd29yZCBmb3IgdXNlciAnJFVzZXJOYW1lJyB0byAnKioqKionIiAjIDopDQogICAgICAgIChbQURTSV0gIldpbk5UOi8vLi8kVXNlck5hbWUiKS5TZXRQYXNzd29yZCgkUGFzc3dvcmQpDQogICAgfQ0KICAgIGVsc2Ugew0KICAgICAgICBXcml0ZS1Mb2dXYXJuaW5nICJZb3UgYXJlIHRyeWluZyB0byBjaGFuZ2UgcGFzc3dvcmQgZm9yIHVzZXIgJyRVc2VyTmFtZScuIFRvIGRvIHRoaXMgcGxlYXNlIHJ1biB0aGUgY29tbWFuZCBhZ2FpbiB3aXRoIC1Gb3JjZSBwYXJhbWV0ZXIuIg0KICAgICAgICAkVXNlckFjY291bnQNCiAgICB9DQp9DQoNCg0KDQpGdW5jdGlvbiBJbnN0YWxsLVJvbGVQcmltYXJ5RG9tYWluQ29udHJvbGxlcg0Kew0KPCMNCi5TWU5PUFNJUw0KQ29uZmlndXJlIG5vZGUncyBuZXR3b3JrIGFkYXB0ZXJzLg0KQ3JlYXRlIGZpcnN0IGRvbWFpbiBjb250cm9sbGVyIGluIHRoZSBmb3Jlc3QuDQoNCi5FWEFNUExFDQpQUz4gSW5zdGFsbC1Sb2xlUHJpbWFyeURvbWFpbkNvbnRyb2xsZXIgLURvbWFpbk5hbWUgYWNtZS5sb2NhbCAtU2FmZU1vZGVQYXNzd29yZCAiUEBzc3cwcmQiDQoNCkluc3RhbGwgRE5TIGFuZCBBRERTLCBjcmVhdGUgZm9yZXN0IGFuZCBkb21haW4gJ2FjbWUubG9jYWwnLg0KU2V0IERDIHJlY292ZXJ5IG1vZGUgcGFzc3dvcmQgdG8gJ1BAc3N3MHJkJy4NCiM+DQoJDQoJcGFyYW0NCgkoDQoJCVtTdHJpbmddDQoJCSMgTmV3IGRvbWFpbiBuYW1lLg0KCQkkRG9tYWluTmFtZSwNCgkJDQoJCVtTdHJpbmddDQoJCSMgRG9tYWluIGNvbnRyb2xsZXIgcmVjb3ZlcnkgbW9kZSBwYXNzd29yZC4NCgkJJFNhZmVNb2RlUGFzc3dvcmQNCgkpDQoNCgl0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQ0KDQogICAgICAgICMgQWRkIHJlcXVpcmVkIHdpbmRvd3MgZmVhdHVyZXMNCglBZGQtV2luZG93c0ZlYXR1cmVXcmFwcGVyIGANCgkJLU5hbWUgIkROUyIsIkFELURvbWFpbi1TZXJ2aWNlcyIsIlJTQVQtREZTLU1nbXQtQ29uIiBgDQoJCS1JbmNsdWRlTWFuYWdlbWVudFRvb2xzIGANCiAgICAgICAgLU5vdGlmeVJlc3RhcnQNCg0KDQoJV3JpdGUtTG9nICJDcmVhdGluZyBmaXJzdCBkb21haW4gY29udHJvbGxlciAuLi4iDQoJCQ0KCSRTTUFQID0gQ29udmVydFRvLVNlY3VyZVN0cmluZyAtU3RyaW5nICRTYWZlTW9kZVBhc3N3b3JkIC1Bc1BsYWluVGV4dCAtRm9yY2UNCgkJDQoJSW5zdGFsbC1BRERTRm9yZXN0IGANCgkJLURvbWFpbk5hbWUgJERvbWFpbk5hbWUgYA0KCQktU2FmZU1vZGVBZG1pbmlzdHJhdG9yUGFzc3dvcmQgJFNNQVAgYA0KCQktRG9tYWluTW9kZSBEZWZhdWx0IGANCgkJLUZvcmVzdE1vZGUgRGVmYXVsdCBgDQoJCS1Ob1JlYm9vdE9uQ29tcGxldGlvbiBgDQoJCS1Gb3JjZSBgDQoJCS1FcnJvckFjdGlvbiBTdG9wIHwgT3V0LU51bGwNCg0KCVdyaXRlLUhvc3QgIldhaXRpbmcgZm9yIHJlYm9vdCAuLi4iCQkNCiMJU3RvcC1FeGVjdXRpb24gLUV4aXRDb2RlIDMwMTAgLUV4aXRTdHJpbmcgIkNvbXB1dGVyIG11c3QgYmUgcmVzdGFydGVkIHRvIGZpbmlzaCBkb21haW4gY29udHJvbGxlciBwcm9tb3Rpb24uIg0KIwlXcml0ZS1Mb2cgIlJlc3RhcmluZyBjb21wdXRlciAuLi4iDQojCVJlc3RhcnQtQ29tcHV0ZXIgLUZvcmNlDQp9DQo="
],
"Commands": [
{
"Name": "Import-Module",
"Arguments": {
"Name": "CoreFunctions"
}
},
{
"Name": "Install-RolePrimaryDomainController",
"Arguments": {
"DomainName": "$domain",
"SafeModePassword": "$recoveryPassword"
}
}
],
"RebootOnCompletion": 1
}

23
conductor/data/templates/agent/CreateSecondaryDC.template Normal file
View File

@ -0,0 +1,23 @@
{
"Scripts": [
"RnVuY3Rpb24gSW5zdGFsbC1Sb2xlU2Vjb25kYXJ5RG9tYWluQ29udHJvbGxlcg0Kew0KPCMNCi5TWU5PUFNJUw0KSW5zdGFsbCBhZGRpdGlvbmFsIChzZWNvbmRhcnkpIGRvbWFpbiBjb250cm9sbGVyLg0KDQojPg0KCXBhcmFtDQoJKA0KCQlbU3RyaW5nXQ0KCQkjIERvbWFpbiBuYW1lIHRvIGpvaW4gdG8uDQoJCSREb21haW5OYW1lLA0KCQkNCgkJW1N0cmluZ10NCgkJIyBEb21haW4gdXNlciB3aG8gaXMgYWxsb3dlZCB0byBqb2luIGNvbXB1dGVyIHRvIGRvbWFpbi4NCgkJJFVzZXJOYW1lLA0KCQkNCgkJW1N0cmluZ10NCgkJIyBVc2VyJ3MgcGFzc3dvcmQuDQoJCSRQYXNzd29yZCwNCgkJDQoJCVtTdHJpbmddDQoJCSMgRG9tYWluIGNvbnRyb2xsZXIgcmVjb3ZlcnkgbW9kZSBwYXNzd29yZC4NCgkJJFNhZmVNb2RlUGFzc3dvcmQNCgkpDQoNCgl0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQ0KCQ0KCSRDcmVkZW50aWFsID0gTmV3LUNyZWRlbnRpYWwgLVVzZXJOYW1lICIkRG9tYWluTmFtZVwkVXNlck5hbWUiIC1QYXNzd29yZCAkUGFzc3dvcmQNCgkJDQoJIyBBZGQgcmVxdWlyZWQgd2luZG93cyBmZWF0dXJlcw0KCUFkZC1XaW5kb3dzRmVhdHVyZVdyYXBwZXIgYA0KCQktTmFtZSAiRE5TIiwiQUQtRG9tYWluLVNlcnZpY2VzIiwiUlNBVC1ERlMtTWdtdC1Db24iIGANCgkJLUluY2x1ZGVNYW5hZ2VtZW50VG9vbHMgYA0KICAgICAgICAgICAgICAgIC1Ob3RpZnlSZXN0YXJ0DQoJCQ0KCQ0KICAgICAgICBXcml0ZS1Mb2cgIkFkZGluZyBzZWNvbmRhcnkgZG9tYWluIGNvbnRyb2xsZXIgLi4uIg0KICAgIA0KCSRTTUFQID0gQ29udmVydFRvLVNlY3VyZVN0cmluZyAtU3RyaW5nICRTYWZlTW9kZVBhc3N3b3JkIC1Bc1BsYWluVGV4dCAtRm9yY2UNCg0KCUluc3RhbGwtQUREU0RvbWFpbkNvbnRyb2xsZXIgYA0KCQktRG9tYWluTmFtZSAkRG9tYWluTmFtZSBgDQoJCS1TYWZlTW9kZUFkbWluaXN0cmF0b3JQYXNzd29yZCAkU01BUCBgDQoJCS1DcmVkZW50aWFsICRDcmVkZW50aWFsIGANCgkJLU5vUmVib290T25Db21wbGV0aW9uIGANCgkJLUZvcmNlIGANCgkJLUVycm9yQWN0aW9uIFN0b3AgfCBPdXQtTnVsbA0KDQoJV3JpdGUtTG9nICJXYWl0aW5nIGZvciByZXN0YXJ0IC4uLiINCiMJU3RvcC1FeGVjdXRpb24gLUV4aXRDb2RlIDMwMTAgLUV4aXRTdHJpbmcgIkNvbXB1dGVyIG11c3QgYmUgcmVzdGFydGVkIHRvIGZpbmlzaCBkb21haW4gY29udHJvbGxlciBwcm9tb3Rpb24uIg0KIwlXcml0ZS1Mb2cgIlJlc3RhcnRpbmcgY29tcHV0ZXIgLi4uIg0KIwlSZXN0YXJ0LUNvbXB1dGVyIC1Gb3JjZQ0KfQ0K"
],
"Commands": [
{
"Name": "Import-Module",
"Arguments": {
"Name": "CoreFunctions"
}
},
{
"Name": "Install-RoleSecondaryDomainController",
"Arguments": {
"DomainName": "$domain",
"UserName": "Administrator",
"Password": "$domainPassword",
"SafeModePassword": "$recoveryPassword"
}
}
],
"RebootOnCompletion": 1
}

12
conductor/data/templates/agent/InstallIIS.template Normal file
View File

@ -0,0 +1,12 @@
{
"Scripts": [
"ZnVuY3Rpb24gSW5zdGFsbC1XZWJTZXJ2ZXIgew0KICAgIEltcG9ydC1Nb2R1bGUgU2VydmVyTWFuYWdlcg0KICAgIEluc3RhbGwtV2luZG93c0ZlYXR1cmUgV2ViLVNlcnZlciAtSW5jbHVkZU1hbmFnZW1lbnRUb29scw0KfQ0K"
],
"Commands": [
{
"Name": "Install-WebServer",
"Arguments": {}
}
],
"RebootOnCompletion": 0
}

27
conductor/data/templates/agent/JoinDomain.template Normal file
View File

@ -0,0 +1,27 @@
{
"Scripts": [],
"Commands": [
{
"Name": "Import-Module",
"Arguments": {
"Name": "CoreFunctions"
}
},
{
"Name": "Set-NetworkAdapterConfiguration",
"Arguments": {
"FirstAvailable": true,
"DNSServer": "$dnsIp"
}
},
{
"Name": "Join-Domain",
"Arguments": {
"DomainName": "$domain",
"Username": "Administrator",
"Password": "$domainPassword"
}
}
],
"RebootOnCompletion": 1
}

22
conductor/data/templates/agent/SetPassword.template Normal file
View File

@ -0,0 +1,22 @@
{
"Scripts": [
"RnVuY3Rpb24gU2V0LUxvY2FsVXNlclBhc3N3b3JkIHsNCiAgICBwYXJhbSAoDQogICAgICAgIFtTdHJpbmddICRVc2VyTmFtZSwNCiAgICAgICAgW1N0cmluZ10gJFBhc3N3b3JkLA0KICAgICAgICBbU3dpdGNoXSAkRm9yY2UNCiAgICApDQogICAgDQogICAgdHJhcCB7IFN0b3AtRXhlY3V0aW9uICRfIH0NCiAgICANCiAgICBpZiAoKEdldC1XbWlPYmplY3QgV2luMzJfVXNlckFjY291bnQgLUZpbHRlciAiTG9jYWxBY2NvdW50ID0gJ1RydWUnIEFORCBOYW1lPSckVXNlck5hbWUnIikgLWVxICRudWxsKSB7DQogICAgICAgIHRocm93ICJVbmFibGUgdG8gZmluZCBsb2NhbCB1c2VyIGFjY291bnQgJyRVc2VyTmFtZSciDQogICAgfQ0KICAgIA0KICAgIGlmICgkRm9yY2UpIHsNCiAgICAgICAgV3JpdGUtTG9nICJDaGFuZ2luZyBwYXNzd29yZCBmb3IgdXNlciAnJFVzZXJOYW1lJyB0byAnKioqKionIiAjIDopDQogICAgICAgIChbQURTSV0gIldpbk5UOi8vLi8kVXNlck5hbWUiKS5TZXRQYXNzd29yZCgkUGFzc3dvcmQpDQogICAgfQ0KICAgIGVsc2Ugew0KICAgICAgICBXcml0ZS1Mb2dXYXJuaW5nICJZb3UgYXJlIHRyeWluZyB0byBjaGFuZ2UgcGFzc3dvcmQgZm9yIHVzZXIgJyRVc2VyTmFtZScuIFRvIGRvIHRoaXMgcGxlYXNlIHJ1biB0aGUgY29tbWFuZCBhZ2FpbiB3aXRoIC1Gb3JjZSBwYXJhbWV0ZXIuIg0KICAgICAgICAkVXNlckFjY291bnQNCiAgICB9DQp9DQoNCg0KDQpGdW5jdGlvbiBJbnN0YWxsLVJvbGVQcmltYXJ5RG9tYWluQ29udHJvbGxlcg0Kew0KPCMNCi5TWU5PUFNJUw0KQ29uZmlndXJlIG5vZGUncyBuZXR3b3JrIGFkYXB0ZXJzLg0KQ3JlYXRlIGZpcnN0IGRvbWFpbiBjb250cm9sbGVyIGluIHRoZSBmb3Jlc3QuDQoNCi5FWEFNUExFDQpQUz4gSW5zdGFsbC1Sb2xlUHJpbWFyeURvbWFpbkNvbnRyb2xsZXIgLURvbWFpbk5hbWUgYWNtZS5sb2NhbCAtU2FmZU1vZGVQYXNzd29yZCAiUEBzc3cwcmQiDQoNCkluc3RhbGwgRE5TIGFuZCBBRERTLCBjcmVhdGUgZm9yZXN0IGFuZCBkb21haW4gJ2FjbWUubG9jYWwnLg0KU2V0IERDIHJlY292ZXJ5IG1vZGUgcGFzc3dvcmQgdG8gJ1BAc3N3MHJkJy4NCiM+DQoJDQoJcGFyYW0NCgkoDQoJCVtTdHJpbmddDQoJCSMgTmV3IGRvbWFpbiBuYW1lLg0KCQkkRG9tYWluTmFtZSwNCgkJDQoJCVtTdHJpbmddDQoJCSMgRG9tYWluIGNvbnRyb2xsZXIgcmVjb3ZlcnkgbW9kZSBwYXNzd29yZC4NCgkJJFNhZmVNb2RlUGFzc3dvcmQNCgkpDQoNCgl0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQ0KDQogICAgICAgICMgQWRkIHJlcXVpcmVkIHdpbmRvd3MgZmVhdHVyZXMNCglBZGQtV2luZG93c0ZlYXR1cmVXcmFwcGVyIGANCgkJLU5hbWUgIkROUyIsIkFELURvbWFpbi1TZXJ2aWNlcyIsIlJTQVQtREZTLU1nbXQtQ29uIiBgDQoJCS1JbmNsdWRlTWFuYWdlbWVudFRvb2xzIGANCiAgICAgICAgLU5vdGlmeVJlc3RhcnQNCg0KDQoJV3JpdGUtTG9nICJDcmVhdGluZyBmaXJzdCBkb21haW4gY29udHJvbGxlciAuLi4iDQoJCQ0KCSRTTUFQID0gQ29udmVydFRvLVNlY3VyZVN0cmluZyAtU3RyaW5nICRTYWZlTW9kZVBhc3N3b3JkIC1Bc1BsYWluVGV4dCAtRm9yY2UNCgkJDQoJSW5zdGFsbC1BRERTRm9yZXN0IGANCgkJLURvbWFpbk5hbWUgJERvbWFpbk5hbWUgYA0KCQktU2FmZU1vZGVBZG1pbmlzdHJhdG9yUGFzc3dvcmQgJFNNQVAgYA0KCQktRG9tYWluTW9kZSBEZWZhdWx0IGANCgkJLUZvcmVzdE1vZGUgRGVmYXVsdCBgDQoJCS1Ob1JlYm9vdE9uQ29tcGxldGlvbiBgDQoJCS1Gb3JjZSBgDQoJCS1FcnJvckFjdGlvbiBTdG9wIHwgT3V0LU51bGwNCg0KCVdyaXRlLUhvc3QgIldhaXRpbmcgZm9yIHJlYm9vdCAuLi4iCQkNCiMJU3RvcC1FeGVjdXRpb24gLUV4aXRDb2RlIDMwMTAgLUV4aXRTdHJpbmcgIkNvbXB1dGVyIG11c3QgYmUgcmVzdGFydGVkIHRvIGZpbmlzaCBkb21haW4gY29udHJvbGxlciBwcm9tb3Rpb24uIg0KIwlXcml0ZS1Mb2cgIlJlc3RhcmluZyBjb21wdXRlciAuLi4iDQojCVJlc3RhcnQtQ29tcHV0ZXIgLUZvcmNlDQp9DQo="
],
"Commands": [
{
"Name": "Import-Module",
"Arguments": {
"Name": "CoreFunctions"
}
},
{
"Name": "Set-LocalUserPassword",
"Arguments": {
"UserName": "Administrator",
"Password": "$adminPassword",
"Force": true
}
}
],
"RebootOnCompletion": 0
}

62
conductor/data/templates/cf/Windows.template Normal file
View File

@ -0,0 +1,62 @@
{
"AWSTemplateFormatVersion" : "2010-09-09",
"Description" : "",
"Parameters" : {
"KeyName" : {
"Description" : "Name of an existing Amazon EC2 key pair for RDP access",
"Type" : "String",
"Default" : "keero_key"
},
"InstanceType" : {
"Description" : "Amazon EC2 instance type",
"Type" : "String",
"Default" : "m1.medium",
"AllowedValues" : [ "m1.small", "m1.medium", "m1.large" ]
},
"ImageName" : {
"Description" : "Image name",
"Type" : "String",
"Default" : "ws-2012-full-agent",
"AllowedValues" : [ "ws-2012-full", "ws-2012-core", "ws-2012-full-agent" ]
}
},
"Resources" : {
"IAMUser" : {
"Type" : "AWS::IAM::User",
"Properties" : {
"Path": "/",
"Policies": [{
"PolicyName": "root",
"PolicyDocument": { "Statement":[{
"Effect": "Allow",
"Action": "CloudFormation:DescribeStackResource",
"Resource": "*"
}]}
}]
}
},
"IAMUserAccessKey" : {
"Type" : "AWS::IAM::AccessKey",
"Properties" : {
"UserName" : {"Ref": "IAMUser"}
}
},
"$instanceName": {
"Type" : "AWS::EC2::Instance",
"Properties": {
"InstanceType" : { "Ref" : "InstanceType" },
"ImageId" : { "Ref" : "ImageName" },
"KeyName" : { "Ref" : "KeyName" },
"UserData": "$userData"
}
}
},
"Outputs" : {
}
}

199
conductor/data/workflows/AD.xml Normal file
View File

@ -0,0 +1,199 @@
<workflow>
<rule match="$.services.activeDirectories[?(@.domain)].units[?(not @.isMaster)]">
<set path="domain">
<select path="::domain"/>
</set>
</rule>
<rule match="$.services.activeDirectories[*].units[?(@.state.instanceName is None)]">
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Creating instance <select path="name"/></parameter>
</report>
<update-cf-stack template="Windows">
<parameter name="mappings">
<map>
<mapping name="instanceName">
<select path="name"/>
</mapping>
<mapping name="userData">
<prepare_user_data/>
</mapping>
</map>
</parameter>
<parameter name="arguments">
<map>
<argument name="KeyName">keero-linux-keys</argument>
<argument name="InstanceType">m1.medium</argument>
<argument name="ImageName">ws-2012-full-agent</argument>
</map>
</parameter>
<success>
<set path="state.instanceName"><select path="name"/></set>
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Instance <select path="name"/> created</parameter>
</report>
</success>
</update-cf-stack>
</rule>
<rule match="$.services.activeDirectories[*].units[?(@.state.instanceName and @.adminPassword and @.adminPassword != @.state.adminPassword)]">
<send-command template="SetPassword">
<parameter name="host">
<select path="name"/>
</parameter>
<parameter name="mappings">
<map>
<mapping name="adminPassword">
<select path="adminPassword"/>
</mapping>
</map>
</parameter>
<success>
<set path="state.adminPassword">
<select path="adminPassword"/>
</set>
</success>
</send-command>
</rule>
<rule match="$.services.activeDirectories[?(@.adminPassword and @.adminPassword != @.state.domainAdminPassword)].units[?(@.state.instanceName and @.isMaster)]">
<send-command template="SetPassword">
<parameter name="host">
<select path="name"/>
</parameter>
<parameter name="mappings">
<map>
<mapping name="adminPassword">
<select path="::adminPassword"/>
</mapping>
</map>
</parameter>
<success>
<set path="::state.domainAdminPassword">
<select path="::adminPassword"/>
</set>
</success>
</send-command>
</rule>
<rule match="$.services.activeDirectories[?(@.state.primaryDc is None)].units[?(@.state.instanceName and @.isMaster)]">
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Creating Primary Domain Controller on unit <select path="name"/></parameter>
</report>
<send-command template="CreatePrimaryDC">
<parameter name="host">
<select path="name"/>
</parameter>
<parameter name="mappings">
<map>
<mapping name="domain">
<select path="::domain"/>
</mapping>
<mapping name="recoveryPassword">
<select path="recoveryPassword"/>
</mapping>
</map>
</parameter>
<success>
<set path="::state.primaryDc"><select path="name"/></set>
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Primary Domain Controller created</parameter>
</report>
</success>
</send-command>
</rule>
<rule match="$.services.activeDirectories[?(@.state.primaryDc and not @.state.primaryDcIp)].units[?(@.state.instanceName and @.isMaster)]">
<send-command template="AskDnsIp" result="ip">
<parameter name="host">
<select path="name"/>
</parameter>
<success>
<set path="::state.primaryDcIp">
<select source="ip" path="0.Result.0"/>
</set>
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">DNS IP = <select source="ip" path="0.Result.0"/></parameter>
</report>
</success>
</send-command>
</rule>
<rule match="$..units[?(@.state.instanceName and @.domain and @.domain != @.state.domain)]">
<set path="#unit">
<select/>
</set>
<rule>
<parameter name="match">/$.services.activeDirectories[?(@.domain == '<select path="domain"/>' and @.state.primaryDcIp)]</parameter>
<send-command template="JoinDomain">
<parameter name="host">
<select path="name" source="unit"/>
</parameter>
<parameter name="mappings">
<map>
<mapping name="domain">
<select path="domain"/>
</mapping>
<mapping name="domainPassword">
<select path="adminPassword"/>
</mapping>
<mapping name="dnsIp">
<select path="state.primaryDcIp"/>
</mapping>
</map>
</parameter>
<success>
<set path="state.domain" target="unit">
<select path="domain"/>
</set>
<report entity="unit">
<parameter name="id"><select path="id" source="unit"/></parameter>
<parameter name="text">Unit <select path="name" source="unit"/> has joined domain <select path="domain"/></parameter>
</report>
</success>
</send-command>
</rule>
</rule>
<rule match="$.services.activeDirectories[*].units[?(@.state.domain and not @.isMaster and not @.state.installed)]">
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Creating Secondary Domain Controller on unit <select path="name"/></parameter>
</report>
<send-command template="CreateSecondaryDC">
<parameter name="host">
<select path="name"/>
</parameter>
<parameter name="mappings">
<map>
<mapping name="recoveryPassword">
<select path="recoveryPassword"/>
</mapping>
<mapping name="domainPassword">
<select path="::adminPassword"/>
</mapping>
</map>
</parameter>
<success>
<set path="state.installed"><true/></set>
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Secondary Domain Controller created</parameter>
</report>
<report entity="service">
<parameter name="id"><select path="::id"/></parameter>
<parameter name="text">Domain <select path="::domain"/> created</parameter>
</report>
</success>
</send-command>
</rule>
</workflow>

60
conductor/data/workflows/IIS.xml Normal file
View File

@ -0,0 +1,60 @@
<workflow>
<rule match="$.services.webServers[?(@.domain)].units[*]">
<set path="domain">
<select path="::domain"/>
</set>
</rule>
<rule match="$.services.webServers[*].units[?(@.state.instanceName is None)]">
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Creating instance <select path="name"/></parameter>
</report>
<update-cf-stack template="Windows">
<parameter name="mappings">
<map>
<mapping name="instanceName">
<select path="name"/>
</mapping>
<mapping name="userData">
<prepare_user_data/>
</mapping>
</map>
</parameter>
<parameter name="arguments">
<map>
<argument name="KeyName">keero-linux-keys</argument>
<argument name="InstanceType">m1.medium</argument>
<argument name="ImageName">ws-2012-full-agent</argument>
</map>
</parameter>
<success>
<set path="state.instanceName"><select path="name"/></set>
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Instance <select path="name"/> created</parameter>
</report>
</success>
</update-cf-stack>
</rule>
<rule match="$.services.webServers[*].units[?(@.state.instanceName and not @.state.iisInstalled)]">
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">Creating IIS Web Server on unit <select path="name"/></parameter>
</report>
<send-command template="InstallIIS">
<parameter name="host">
<select path="name"/>
</parameter>
<success>
<set path="state.iisInstalled"><true/></set>
<report entity="unit">
<parameter name="id"><select path="id"/></parameter>
<parameter name="text">IIS <select path="name"/> has started</parameter>
</report>
</success>
</send-command>
</rule>
</workflow>

5
conductor/etc/app.config Normal file
View File

@ -0,0 +1,5 @@
[rabbitmq]
host = localhost
vhost = keero
login = keero
password = keero

41
conductor/test.json Normal file
View File

@ -0,0 +1,41 @@
{
"name": "MyDataCenter",
"id": "adc6d143f9584d10808c7ef4d07e4802",
"services": {
"activeDirectories": [
{
"id": "9571747991184642B95F430A014616F9",
"domain": "acme.loc",
"adminPassword": "SuperP@ssw0rd!",
"units": [
{
"id": "273c9183b6e74c9c9db7fdd532c5eb25",
"name": "dc01",
"isMaster": true,
"recoveryPassword": "2SuperP@ssw0rd2"
},
{
"id": "377c6f16d17a416791f80724dab360c6",
"name": "dc02",
"isMaster": false,
"adminPassword": "SuperP@ssw0rd",
"recoveryPassword": "2SuperP@ssw0rd2"
}
]
}
],
"webServers": [
{
"id": "e9657ceef84a4e669e31795040080262",
"domain": "acme.loc",
"units": [
{
"id": "e6f9cfd07ced48fba64e6bd9e65aba64",
"name": "iis01",
"adminPassword": "SuperP@ssw0rd"
}
]
}
]
}
}

154
conductor/tools/install_venv.py Normal file
View File

@ -0,0 +1,154 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 OpenStack LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Installation script for Glance's development virtualenv
"""
import os
import subprocess
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
VENV = os.path.join(ROOT, '.venv')
PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires')
TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires')
def die(message, *args):
print >> sys.stderr, message % args
sys.exit(1)
def run_command(cmd, redirect_output=True, check_exit_code=True):
"""
Runs a command in an out-of-process shell, returning the
output of that command. Working directory is ROOT.
"""
if redirect_output:
stdout = subprocess.PIPE
else:
stdout = None
proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout)
output = proc.communicate()[0]
if check_exit_code and proc.returncode != 0:
die('Command "%s" failed.\n%s', ' '.join(cmd), output)
return output
HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'],
check_exit_code=False).strip())
HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'],
check_exit_code=False).strip())
def check_dependencies():
"""Make sure virtualenv is in the path."""
if not HAS_VIRTUALENV:
print 'not found.'
# Try installing it via easy_install...
if HAS_EASY_INSTALL:
print 'Installing virtualenv via easy_install...',
if not run_command(['which', 'easy_install']):
die('ERROR: virtualenv not found.\n\n'
'Balancer development requires virtualenv, please install'
' it using your favorite package management tool')
print 'done.'
print 'done.'
def create_virtualenv(venv=VENV):
"""
Creates the virtual environment and installs PIP only into the
virtual environment
"""
print 'Creating venv...',
run_command(['virtualenv', '-q', '--no-site-packages', VENV])
print 'done.'
print 'Installing pip in virtualenv...',
if not run_command(['tools/with_venv.sh', 'easy_install',
'pip>1.0']).strip():
die("Failed to install pip.")
print 'done.'
def pip_install(*args):
run_command(['tools/with_venv.sh',
'pip', 'install', '--upgrade'] + list(args),
redirect_output=False)
def install_dependencies(venv=VENV):
print 'Installing dependencies with pip (this can take a while)...'
pip_install('pip')
pip_install('-r', PIP_REQUIRES)
pip_install('-r', TEST_REQUIRES)
# Tell the virtual env how to "import glance"
py_ver = _detect_python_version(venv)
pthfile = os.path.join(venv, "lib", py_ver,
"site-packages", "balancer.pth")
f = open(pthfile, 'w')
f.write("%s\n" % ROOT)
def _detect_python_version(venv):
lib_dir = os.path.join(venv, "lib")
for pathname in os.listdir(lib_dir):
if pathname.startswith('python'):
return pathname
raise Exception('Unable to detect Python version')
def print_help():
help = """
Glance development environment setup is complete.
Glance development uses virtualenv to track and manage Python dependencies
while in development and testing.
To activate the Glance virtualenv for the extent of your current shell session
you can run:
$ source .venv/bin/activate
Or, if you prefer, you can run commands in the virtualenv on a case by case
basis by running:
$ tools/with_venv.sh <your command>
Also, make test will automatically use the virtualenv.
"""
print help
def main(argv):
check_dependencies()
create_virtualenv()
install_dependencies()
print_help()
if __name__ == '__main__':
main(sys.argv)

3
conductor/tools/pip-requires Normal file
View File

@ -0,0 +1,3 @@
pika
tornado
jsonpath

8
conductor/tools/test-requires Normal file
View File

@ -0,0 +1,8 @@
unittest2
mock==0.8.0
nose
nose-exclude
nosexcover
#openstack.nose_plugin
pep8==1.0.1
sphinx>=1.1.2

4
conductor/tools/with_venv.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
TOOLS=`dirname $0`
VENV=$TOOLS/../.venv
source $VENV/bin/activate && $@