Merge pull request #183 from loles/vr_riak

Virtual resources improvements

examples/riak/riak_cluster.yaml

@@ -0,0 +1,149 @@
+id: riak_node
+
+resources:
+  - id: riak_service0
+    from: resources/riak_node
+    values:
+      riak_self_name: 'riak0'
+      riak_hostname: 'riak_server0.solar'
+      riak_name: 'riak0@riak_server0.solar'
+      ip: '{{nodes[0]}}::ip'
+      ssh_user: '{{nodes[0]}}::ssh_user'
+      ssh_key: '{{nodes[0]}}::ssh_key'
+
+{% for i in range(1, idx|int) %}
+  - id: riak_service{{i}}
+    from: resources/riak_node
+    values:
+      riak_self_name: 'riak{{i}}'
+      riak_hostname: 'riak_server{{i}}.solar'
+      riak_name: 'riak{{i}}@riak_server{{i}}.solar'
+      join_to: 'riak_service0::riak_name'
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: hosts_file{{i}}
+    from: resources/hosts_file
+    values:
+      hosts:name: 
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_hostname::NO_EVENTS
+        {% endfor %}
+      hosts:ip:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::ip::NO_EVENTS
+        {% endfor %}
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_service_config_http{{i}}
+    from: resources/haproxy_service_config
+    values:
+      listen_port: 8098
+      protocol: 'http'
+      name: 'riak_haproxy_http{{i}}'
+      backends:server:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_hostname
+        {% endfor %}
+      backends:port:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_port_http
+        {% endfor %}
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_service_config_pb{{i}}
+    from: resources/haproxy_service_config
+    values:
+      listen_port: 8087
+      protocol: 'tcp'
+      name: 'riak_haproxy_pb{{i}}'
+      backends:server:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_hostname
+        {% endfor %}
+      backends:port:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_port_pb
+        {% endfor %}
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_config{{i}}
+    from: resources/haproxy_config
+    values:
+      config:protocol:
+        - haproxy_service_config_http{{i}}::protocol
+        - haproxy_service_config_pb{{i}}::protocol
+      config:listen_port:
+        - haproxy_service_config_http{{i}}::listen_port
+        - haproxy_service_config_pb{{i}}::listen_port
+      config:name:
+        - haproxy_service_config_http{{i}}::name
+        - haproxy_service_config_pb{{i}}::name
+      config:backends:
+        - haproxy_service_config_http{{i}}::backends
+        - haproxy_service_config_pb{{i}}::backends
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_service{{i}}
+    from: resources/haproxy_service
+    values:
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+
+events:
+{% for i in range(idx|int) %}
+  - type: depends_on
+    parent_action: 'hosts_file{{i}}.run'
+    state: 'success'
+    depend_action: 'riak_service{{i}}.run'
+{% endfor %}
+
+{% for i in range(1, idx|int) %}
+  - type: react_on
+    parent_action: 'riak_service{{i}}.run'
+    state: 'success'
+    depend_action: 'riak_service{{i}}.join'
+
+  - type: react_on
+    parent_action: 'riak_service{{i}}.leave'
+    state: 'success'
+    depend_action: 'riak_service{{i}}.join'
+
+  - type: react_on
+    parent_action: 'riak_service{{i}}.join'
+    state: 'success'
+    depend_action: 'riak_service0.commit'
+{% endfor %}
+
+{% for i in range(1, idx|int) %}
+  - type: depends_on
+    parent_action: 'haproxy_service{{i}}.run'
+    state: 'success'
+    depend_action: 'haproxy_config{{i}}.run'
+
+  - type: react_on
+    parent_action: 'haproxy_config{{i}}.run'
+    state: 'success'
+    depend_action: 'haproxy_service{{i}}.apply_config'
+
+  - type: react_on
+    parent_action: 'haproxy_config{{i}}.update'
+    state: 'success'
+    depend_action: 'haproxy_service{{i}}.apply_config'
+{% endfor %}

solar/solar/core/resource/virtual_resource.py

@@ -22,6 +22,8 @@ from jinja2 import Template, Environment, meta
 from solar.core import provider
 from solar.core import resource
 from solar.core import signals
+from solar.events.api import add_event
+from solar.events.controls import React, Dep
 
 
 def create(name, base_path, args=None, virtual_resource=None):
@@ -53,6 +55,8 @@ def create_resource(name, base_path, args=None, virtual_resource=None):
     if isinstance(base_path, provider.BaseProvider):
         base_path = base_path.directory
 
+    # List args init with empty list. Elements will be added later
+    args = {key: (value if not isinstance(value, list) else []) for key, value in args.items()}
     r = resource.Resource(
         name, base_path, args=args, tags=[], virtual_resource=virtual_resource
     )
@@ -60,29 +64,13 @@ def create_resource(name, base_path, args=None, virtual_resource=None):
 
 
 def create_virtual_resource(vr_name, template):
-    resources = template['resources']
-    connections = []
-    created_resources = []
-
-    cwd = os.getcwd()
-    for r in resources:
-        name = r['id']
-        base_path = os.path.join(cwd, r['from'])
-        args = r['values']
-        new_resources = create(name, base_path, args, vr_name)
-        created_resources += new_resources
-
-        if not is_virtual(base_path):
-            for key, arg in args.items():
-                if isinstance(arg, basestring) and '::' in arg:
-                    emitter, src = arg.split('::')
-                    connections.append((emitter, name, {src: key}))
-
-        for emitter, reciver, mapping in connections:
-            emitter = r.load(emitter)
-            reciver = r.load(reciver)
-            signals.connect(emitter, reciver, mapping)
+    template_resources = template['resources']
+    template_events = template.get('events', {})
 
+    created_resources = create_resources(template_resources)
+    events = parse_events(template_events)
+    for event in events:
+        add_event(event)
     return created_resources
 
 
@@ -92,13 +80,16 @@ def _compile_file(name, path, kwargs):
 
     inputs = get_inputs(content)
     template = _get_template(name, content, kwargs, inputs)
+    with open('/tmp/compiled', 'w') as c:
+        c.write(template)
     return template
 
 
 def get_inputs(content):
-    env = Environment()
+    env = Environment(trim_blocks=True, lstrip_blocks=True)
+    jinja_globals = env.globals.keys()
     ast = env.parse(content)
-    return meta.find_undeclared_variables(ast)
+    return meta.find_undeclared_variables(ast) - set(jinja_globals)
 
 
 def _get_template(name, content, kwargs, inputs):
@@ -108,10 +99,79 @@ def _get_template(name, content, kwargs, inputs):
             missing.append(input)
     if missing:
         raise Exception('[{0}] Validation error. Missing data in input: {1}'.format(name, missing))
-    template = Template(content)
+    template = Template(content, trim_blocks=True, lstrip_blocks=True)
     template = template.render(str=str, zip=zip, **kwargs)
     return template
 
 
 def is_virtual(path):
     return os.path.isfile(path)
+
+
+def create_resources(resources):
+    created_resources = []
+    cwd = os.getcwd()
+    for r in resources:
+        resource_name = r['id']
+        base_path = os.path.join(cwd, r['from'])
+        args = r['values']
+        new_resources = create(resource_name, base_path, args)
+        created_resources += new_resources
+
+        if not is_virtual(base_path):
+            add_connections(resource_name, args)
+    return created_resources
+
+
+def parse_events(events):
+    parsed_events = []
+    for event in events:
+        event_type = event['type']
+        parent, parent_action = event['parent_action'].split('.')
+        child, child_action = event['depend_action'].split('.')
+        state = event['state']
+        if event_type == Dep.etype:
+            event = Dep(parent, parent_action, state, child, child_action)
+        elif event_type == React.etype:
+            event = React(parent, parent_action, state, child, child_action)
+        else:
+            raise Exception('Invalid event type: {0}'.format(event_type))
+        parsed_events.append(event)
+    return parsed_events
+
+
+def add_connections(resource_name, args):
+    connections = []
+    for receiver_input, arg in args.items():
+        if isinstance(arg, list):
+            for item in arg:
+                c = parse_connection(resource_name, receiver_input, item)
+                connections.append(c)
+        else:
+           c = parse_connection(resource_name, receiver_input, arg)
+           connections.append(c)
+
+    connections = [c for c in connections if c is not None]
+    for c in connections:
+        parent = resource.load(c['parent'])
+        child = resource.load(c['child'])
+        events = c['events']
+        mapping = {c['parent_input'] : c['child_input']}
+        signals.connect(parent, child, mapping, events)
+
+
+def parse_connection(receiver, receiver_input, element):
+    if isinstance(element, basestring) and '::' in element:
+        emitter, src = element.split('::', 1)
+        try:
+            src, events = src.split('::')
+            if events == 'NO_EVENTS':
+                events = False
+        except ValueError:
+            events = None
+        return {'child': receiver,
+                'child_input': receiver_input,
+                'parent' : emitter,
+                'parent_input': src,
+                'events' : events
+                }

solar/solar/interfaces/db/redis_graph_db.py

@@ -148,12 +148,13 @@ class RedisGraphDB(BaseGraphDB):
         """Fetch element with given name and collection type."""
 
         try:
-            item = self._r.get(self._make_collection_key(collection, name))
+            collection_key = self._make_collection_key(collection, name)
+            item = self._r.get(collection_key)
             if not item and return_empty:
                 return item
             return json.loads(item)
         except TypeError:
-            raise KeyError
+            raise KeyError(collection_key)
 
     def delete(self, name, collection=BaseGraphDB.DEFAULT_COLLECTION):
         keys = self._r.keys(self._make_collection_key(collection, name))

solar/solar/test/conftest.py

@@ -11,12 +11,30 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import os
 
+import pytest
 
-from pytest import fixture
-
+from solar.core.resource import Resource
 from solar.interfaces import db
 
+@pytest.fixture
+def resources():
+    base_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        'resource_fixtures')
+
+    node_path = os.path.join(base_path, 'node')
+    node1 = Resource('node1', node_path, args={'ip':'10.0.0.1'})
+    node2 = Resource('node2', node_path, args={'ip':'10.0.0.2'})
+
+    base_service_path = os.path.join(base_path, 'base_service')
+    service1 = Resource('service1', base_service_path)
+    return {'node1' : node1,
+            'node2' : node2,
+            'service1': service1
+           }
+
 
 def pytest_configure():
     if db.CURRENT_DB == 'redis_graph_db':
@@ -27,7 +45,7 @@ def pytest_configure():
         db.DB = db.get_db(backend=db.CURRENT_DB)
 
 
-@fixture(autouse=True)
+@pytest.fixture(autouse=True)
 def cleanup(request):
 
     def fin():

solar/solar/test/resource_fixtures/base_service/meta.yaml

@@ -0,0 +1,13 @@
+id: base_service
+handler: ansible
+version: 1.0.0
+input:
+  ip:
+    schema: str!
+    value:
+  servers:
+    schema: [str]
+    value: []
+  alias:
+    schema: str
+    value:

solar/solar/test/resource_fixtures/node/meta.yaml

@@ -0,0 +1,7 @@
+id: node
+handler: ansible
+version: 1.0.0
+input:
+  ip:
+    schema: str!
+    value:

solar/solar/test/resource_fixtures/nodes.yaml.tmpl

@@ -0,0 +1,10 @@
+id: simple_multinode
+resources:
+  - id: node1
+    from: {resource_path}
+    values:
+      ip: '10.0.0.3'
+  - id: node2
+    from: {resource_path}
+    values:
+      ip: '10.0.0.4'

solar/solar/test/test_virtual_resource.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import os
+from StringIO import StringIO
+
+import pytest
+import yaml
+
+from solar.events.controls import React, Dep
+from solar.core.resource import virtual_resource as vr
+
+
+@pytest.fixture
+def good_events():
+    events = '''
+  - type: depends_on
+    parent_action: 'service1.run'
+    state: 'success'
+    depend_action: 'config1.run'
+  - type: react_on
+    parent_action: 'config1.run'
+    state: 'success'
+    depend_action: 'service1.apply_config'
+'''
+    return yaml.load(StringIO(events))
+
+@pytest.fixture
+def bad_event_type():
+    events = '''
+  - type: skip
+    parent_action: 'service1.run'
+    state: 'success'
+    depend_action: 'config1.run'
+'''
+    return yaml.load(StringIO(events))
+
+
+def test_create_resource():
+    node_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        'resource_fixtures', 'node')
+    resources = vr.create('node1', node_path)
+    assert len(resources) == 1
+    assert resources[0].name == 'node1'
+
+def test_create_virtual_resource(tmpdir):
+    base_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        'resource_fixtures')
+    vr_tmpl_path = os.path.join(base_path, 'nodes.yaml.tmpl')
+    node_resource_path = os.path.join(base_path, 'node')
+    with open(vr_tmpl_path) as f:
+        vr_data = f.read().format(resource_path=node_resource_path)
+    vr_file = tmpdir.join('nodes.yaml')
+    vr_file.write(vr_data)
+    resources = vr.create('nodes', str(vr_file))
+    assert len(resources) == 2
+
+
+def test_parse_events(good_events):
+    events =[Dep(parent='service1', parent_action='run',
+                 child='config1', child_action='run',
+                 state='success'),
+             React(parent='config1', parent_action='run',
+                   child='service1', child_action='apply_config',
+                   state='success')]
+    parsed = vr.parse_events(good_events)
+    assert events == parsed
+
+def test_parse_bad_event(bad_event_type):
+    with pytest.raises(Exception) as execinfo:
+        vr.parse_events(bad_event_type)
+    error = 'Invalid event type: skip'
+    assert error == str(execinfo.value)
+
+
+def test_add_connections(mocker, resources):
+    mocked_signals = mocker.patch('solar.core.resource.virtual_resource.signals')
+    args = {'ip': 'node1::ip',
+            'servers': ['node1::ip', 'node2::ip'],
+            'alias': 'ser1'
+           }
+    vr.add_connections('service1', args)
+    assert mocked_signals.connect.call_count == 3
+
+
+def test_parse_connection():
+    correct_connection = {'child': 'host_file',
+     'child_input': 'ip',
+     'parent' : 'node1',
+     'parent_input': 'ip',
+     'events' : None
+    }
+    connection = vr.parse_connection('host_file', 'ip', 'node1::ip')
+    assert correct_connection == connection
+
+def test_parse_connection_disable_events():
+    correct_connection = {'child': 'host_file',
+     'child_input': 'ip',
+     'parent' : 'node1',
+     'parent_input': 'ip',
+     'events' : False
+    }
+    connection = vr.parse_connection('host_file', 'ip', 'node1::ip::NO_EVENTS')
+    assert correct_connection == connection
+
+def test_parse_connection_no_connection():
+    connection = vr.parse_connection('host_file', 'ip', '10.0.0.2')
+    assert None == connection

solar/test-requirements.txt

@@ -1 +1,2 @@
 -r requirements.txt
+pytest-mock