Merge "Add support for rendering jinja template as yaml"

jenkins_jobs/formatter.py

@@ -26,7 +26,7 @@ from jenkins_jobs.local_yaml import CustomLoader
 logger = logging.getLogger(__name__)
 
 
-def deep_format(obj, paramdict, allow_empty=False):
+def deep_format(obj, paramdict, allow_empty=False, template=True):
     """Apply the paramdict via str.format() to all string objects found within
        the supplied obj. Lists and dicts are traversed recursively."""
     # YAML serialisation was originally used to achieve this, but that places
@@ -50,13 +50,26 @@ def deep_format(obj, paramdict, allow_empty=False):
     elif isinstance(obj, list):
         ret = type(obj)()
         for item in obj:
-            ret.append(deep_format(item, paramdict, allow_empty))
+            ret.append(deep_format(item, paramdict,
+                                   allow_empty=allow_empty,
+                                   template=template))
     elif isinstance(obj, dict):
         ret = type(obj)()
         for item in obj:
             try:
-                ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \
-                    deep_format(obj[item], paramdict, allow_empty)
+                # deep_formatting dsl when not a job-template is not necessary
+                # as it will most likely result in keyerror due to trying
+                # to substitute values inside the dsl that do not exist.
+                if item not in ['dsl'] or template:
+                    ret[CustomFormatter(allow_empty).format(item,
+                                                            **paramdict)] = \
+                        deep_format(obj[item], paramdict,
+                                    allow_empty=allow_empty,
+                                    template=template)
+                else:
+                    ret[CustomFormatter(allow_empty).format(item,
+                                                            **paramdict)] = \
+                        obj[item]
             except KeyError as exc:
                 missing_key = exc.args[0]
                 desc = "%s parameter missing to format %s\nGiven:\n%s" % (
@@ -72,7 +85,8 @@ def deep_format(obj, paramdict, allow_empty=False):
     if isinstance(ret, CustomLoader):
         # If we have a CustomLoader here, we've lazily-loaded a template;
         # attempt to format it.
-        ret = deep_format(ret, paramdict, allow_empty=allow_empty)
+        ret = deep_format(ret, paramdict, allow_empty=allow_empty,
+                          template=template)
     return ret
 
 

jenkins_jobs/local_yaml.py

@@ -491,6 +491,17 @@ class YamlIncludeJinja2(YamlIncludeRaw):
         return Jinja2Loader(contents, loader.search_path)
 
 
+class YamlIncludeJinja2AsYaml(YamlIncludeJinja2):
+    yaml_tag = u'!include-jinja2-as-yaml:'
+
+    @classmethod
+    def _from_file(cls, loader, node):
+        contents = cls._open_file(loader, node)
+        if isinstance(contents, LazyLoader):
+            return contents
+        return Jinja2LoaderAsYaml(contents, loader.search_path)
+
+
 class DeprecatedTag(BaseYAMLObject):
 
     @classmethod
@@ -537,6 +548,14 @@ class Jinja2Loader(CustomLoader):
         return self._template.render(kwargs)
 
 
+class Jinja2LoaderAsYaml(Jinja2Loader):
+    """A loader for Jinja2-templated files that renders yaml."""
+
+    def format(self, **kwargs):
+        raw_yaml = super(Jinja2LoaderAsYaml, self).format(**kwargs)
+        return yaml.load(raw_yaml)
+
+
 class CustomLoaderCollection(object):
     """Helper class to format a collection of CustomLoader objects"""
     def __init__(self, sequence):

jenkins_jobs/parser.py

@@ -243,6 +243,16 @@ class YamlParser(object):
             if jobs_glob and not matches(job['name'], jobs_glob):
                 logger.debug("Ignoring job {0}".format(job['name']))
                 continue
+
+            # Attempt to format all parts of the job definition as they might
+            # be using custom loaders.
+            try:
+                job = deep_format(job, job, template=False)
+            except Exception:
+                logging.error(
+                    "Failure formatting job '%s' with itself", job)
+                raise
+
             logger.debug("Expanding job '{0}'".format(job['name']))
             self._formatDescription(job)
             self.jobs.append(job)

tests/yamlparser/fixtures/jinja-as-yaml-include01.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <hudson.model.ParametersDefinitionProperty>
+      <parameterDefinitions>
+        <hudson.model.ChoiceParameterDefinition>
+          <name>TEST_CHOICE</name>
+          <description/>
+          <choices class="java.util.Arrays$ArrayList">
+            <a class="string-array">
+              <string>a</string>
+              <string>b</string>
+              <string>c</string>
+            </a>
+          </choices>
+        </hudson.model.ChoiceParameterDefinition>
+      </parameterDefinitions>
+    </hudson.model.ParametersDefinitionProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml

@@ -0,0 +1,7 @@
+- job:
+    name: test-job-as-yaml
+    parameters:
+        - choice:
+            name: TEST_CHOICE
+            choices:
+                !include-jinja2-as-yaml: jinja-as-yaml-include01.yaml.inc

tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml.inc

@@ -0,0 +1,3 @@
+{% for item in ['a', 'b', 'c'] %}
+- {{ item }}
+{% endfor -%}