Merge "Validate scenarios against schema"

requirements.txt

@@ -14,6 +14,7 @@ oslo.serialization>=1.10.0 # Apache-2.0
 oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0
 psutil<2.0.0,>=1.1.1
 pygal
+pykwalify
 python-glanceclient>=0.18.0
 python-keystoneclient!=1.8.0,>=1.6.0
 python-neutronclient>=2.6.0

shaker/engine/config.py

@@ -26,6 +26,7 @@ from shaker.engine import utils
 IMAGE_BUILDER_TEMPLATES = 'shaker/resources/image_builder_templates/'
 REPORT_TEMPLATES = 'shaker/resources/report_templates/'
 SCENARIOS = 'shaker/scenarios/'
+SCHEMAS = 'shaker/resources/schemas/'
 
 
 class Endpoint(types.String):

shaker/engine/server.py

@@ -224,6 +224,11 @@ def act():
 
         LOG.info('Play scenario: %s', scenario_file_name)
         scenario = utils.read_yaml_file(scenario_file_name)
+
+        schema = utils.read_yaml_file(utils.resolve_relative_path(
+            '%s%s.yaml' % (config.SCHEMAS, 'scenario')))
+        utils.validate_yaml(scenario, schema)
+
         scenario['title'] = scenario.get('title') or scenario_file_name
         scenario['file_name'] = scenario_file_name
 

shaker/engine/utils.py

@@ -18,11 +18,13 @@ import itertools
 import logging as std_logging
 import os
 import random
+import re
 import uuid
 
 from oslo_config import cfg
 from oslo_log import log as logging
-import re
+from pykwalify import core as pykwalify_core
+from pykwalify import errors as pykwalify_errors
 import six
 import yaml
 
@@ -131,6 +133,7 @@ def read_yaml_file(file_name):
     except Exception as e:
         LOG.error('Failed to parse file %(file)s in YAML format: %(err)s',
                   dict(file=file_name, err=e))
+        raise
 
 
 def split_address(address):
@@ -212,3 +215,11 @@ def algebraic_product(**kwargs):
 
 def strict(s):
     return re.sub(r'[^\w\d]+', '_', re.sub(r'\(.+\)', '', s)).lower()
+
+
+def validate_yaml(data, schema):
+    c = pykwalify_core.Core(source_data=data, schema_data=schema)
+    try:
+        c.validate(raise_exception=True)
+    except pykwalify_errors.SchemaError as e:
+        raise Exception('File does not conform to schema: %s' % e)

shaker/resources/schemas/scenario.yaml

@@ -0,0 +1,59 @@
+name: Shaker scenario schema
+type: map
+allowempty: True
+mapping:
+  title:
+    type: str
+  description:
+    type: str
+  deployment:
+    type: map
+    mapping:
+      template:
+        type: str
+      agents:
+        type: any
+      accommodation:
+        type: seq
+        matching: any
+        sequence:
+        - type: str
+          enum: [pair, alone, double_room, single_room, mixed_room, cross_az]
+        - type: map
+          mapping:
+            density:
+              type: number
+            compute_nodes:
+              type: number
+            zones:
+              type: seq
+              sequence:
+              - type: str
+  execution:
+    type: map
+    mapping:
+      progression:
+        type: str
+        enum: [linear, arithmetic, quadratic, geometric]
+      tests:
+        type: seq
+        required: True
+        sequence:
+        - type: map
+          allowempty: True
+          mapping:
+            title:
+              type: str
+            class:
+              type: str
+              required: True
+            method:
+              type: str
+            time:
+              type: int
+              range:
+                min: 1
+            sla:
+              type: seq
+              sequence:
+              - type: str

shaker/scenarios/networking/cross_az/udp_l2_mss8950.yaml

@@ -6,7 +6,7 @@ description:
 
 deployment:
   template: l2.hot
-  vm_accommodation: [pair, single_room, zones: [nova, vcenter], cross_az]
+  accommodation: [pair, single_room, zones: [nova, vcenter], cross_az]
 
 execution:
   progression: quadratic

shaker/tests/test_scenarios.py

@@ -20,6 +20,8 @@ import six
 import testtools
 import yaml
 
+from shaker.engine import utils
+
 
 class TestReport(testtools.TestCase):
 
@@ -31,15 +33,30 @@ class TestReport(testtools.TestCase):
         with opener(file_name, 'r') as content_file:
             return content_file.read()
 
-    def test_yaml_valid(self):
-        for dir_data in os.walk('shaker/scenarios'):
+    def _iterate_files(self, root_path):
+        for dir_data in os.walk(root_path):
             dir_path, dir_names, file_names = dir_data
             for file_name in file_names:
-                if not file_name.endswith('.yaml'):
-                    continue
+                if file_name.endswith('.yaml'):
+                    yield os.path.join(dir_path, file_name)
 
-                cnt = self._read_raw_file(os.path.join(dir_path, file_name))
-                try:
-                    yaml.safe_load(cnt)
-                except Exception as e:
-                    self.fail('File %s is invalid: %s' % (file_name, e))
+    def test_yaml_valid(self):
+        for file_name in self._iterate_files('shaker/scenarios'):
+            cnt = self._read_raw_file(file_name)
+            try:
+                yaml.safe_load(cnt)
+            except Exception as e:
+                self.fail('File %s is invalid: %s' % (file_name, e))
+
+    def test_scenario_schema_conformance(self):
+        scenario_schema_file = 'shaker/resources/schemas/scenario.yaml'
+
+        for file_name in self._iterate_files('shaker/scenarios/'):
+            source_data = utils.read_yaml_file(file_name)
+            schema_data = utils.read_yaml_file(scenario_schema_file)
+
+            try:
+                utils.validate_yaml(source_data, schema_data)
+            except Exception as e:
+                self.fail('Scenario %s does not conform to schema: %s' %
+                          (file_name, e))