Introduce a new '!j2-yaml:' tag

The tag provides Jinja templating capabilities for generating parts of
YAML structures. Two potential use cases are provided as test cases and
are linked in the documentation.

The new tag should also help address some use cases people were asking
about, like here:
https://groups.google.com/g/jenkins-job-builder/c/HkVZVuBDlKM.

Change-Id: I96392e42c3c79a9be0a8f736506908701251dd62

jenkins_jobs/formatter.py

@@ -79,9 +79,12 @@ def deep_format(obj, paramdict, allow_empty=False):
     else:
         ret = obj
     if isinstance(ret, CustomLoader):
-        # If we have a CustomLoader here, we've lazily-loaded a template;
+        # If we have a CustomLoader here, we've lazily-loaded a template
+        # or rendered a template to a piece of YAML;
         # attempt to format it.
-        ret = deep_format(ret, paramdict, allow_empty=allow_empty)
+        ret = deep_format(
+            ret.get_object_to_format(), paramdict, allow_empty=allow_empty
+        )
     return ret
 
 

jenkins_jobs/local_yaml.py

@@ -196,6 +196,24 @@ construct.
 Examples:
 
     .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-string01.yaml
+
+The tag ``!j2-yaml:`` is similar to the ``!j2:`` tag, just that it loads the
+Jinja-rendered string as YAML and embeds it in the calling YAML construct. This
+provides a very flexible and convenient way of generating pieces of YAML
+structures. One of use cases is defining complex YAML structures with much
+simpler configuration, without any duplication.
+
+Examples:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-yaml01.yaml
+
+Another use case is controlling lists dynamically, like conditionally adding
+list elements based on project configuration.
+
+Examples:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/jinja-yaml02.yaml
+
 """
 
 import functools
@@ -369,6 +387,14 @@ class BaseYAMLObject(YAMLObject):
     yaml_dumper = LocalDumper
 
 
+class J2Yaml(BaseYAMLObject):
+    yaml_tag = u"!j2-yaml:"
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        return Jinja2YamlLoader(node.value, loader.search_path)
+
+
 class J2String(BaseYAMLObject):
     yaml_tag = u"!j2:"
 
@@ -572,6 +598,26 @@ class Jinja2Loader(CustomLoader):
         self._template.environment.loader = self._loader
         return self._template.render(kwargs)
 
+    def get_object_to_format(self):
+        return self
+
+
+class LateYamlLoader(CustomLoader):
+    """A loader for data rendered via Jinja2, to be loaded as YAML and then deep formatted."""
+
+    def __init__(self, yaml_str, loader):
+        self._yaml_str = yaml_str
+        self._loader = loader
+
+    def get_object_to_format(self):
+        return load(self._yaml_str, search_path=self._loader._search_path)
+
+
+class Jinja2YamlLoader(Jinja2Loader):
+    def format(self, **kwargs):
+        yaml_str = super(Jinja2YamlLoader, self).format(**kwargs)
+        return LateYamlLoader(yaml_str, self)
+
 
 class CustomLoaderCollection(object):
     """Helper class to format a collection of CustomLoader objects"""

tests/yamlparser/fixtures/jinja-yaml01.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <triggers class="vector">
+    <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+      <spec/>
+      <gerritProjects>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+          <compareType>REG_EXP</compareType>
+          <pattern>a|b|c</pattern>
+          <branches>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+              <compareType>PLAIN</compareType>
+              <pattern>master</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+          </branches>
+          <disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+          <compareType>REG_EXP</compareType>
+          <pattern>d|e|f</pattern>
+          <branches>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+              <compareType>PLAIN</compareType>
+              <pattern>stable</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+          </branches>
+          <disableStrictForbiddenFileVerification>false</disableStrictForbiddenFileVerification>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+      </gerritProjects>
+      <skipVote>
+        <onSuccessful>false</onSuccessful>
+        <onFailed>false</onFailed>
+        <onUnstable>false</onUnstable>
+        <onNotBuilt>false</onNotBuilt>
+      </skipVote>
+      <silentMode>false</silentMode>
+      <silentStartMode>false</silentStartMode>
+      <escapeQuotes>true</escapeQuotes>
+      <dependencyJobsNames/>
+      <commitMessageParameterMode>BASE64</commitMessageParameterMode>
+      <nameAndEmailParameterMode>PLAIN</nameAndEmailParameterMode>
+      <changeSubjectParameterMode>PLAIN</changeSubjectParameterMode>
+      <commentTextParameterMode>BASE64</commentTextParameterMode>
+      <notificationLevel/>
+      <dynamicTriggerConfiguration>false</dynamicTriggerConfiguration>
+      <triggerConfigURL/>
+      <dynamicGerritProjects class="empty-list"/>
+      <triggerInformationAction/>
+      <triggerOnEvents/>
+      <buildStartMessage/>
+      <buildFailureMessage/>
+      <buildSuccessfulMessage/>
+      <buildUnstableMessage/>
+      <buildNotBuiltMessage/>
+      <buildUnsuccessfulFilepath/>
+      <customUrl/>
+      <serverName>__ANY__</serverName>
+    </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+  </triggers>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-yaml01.yaml

@@ -0,0 +1,30 @@
+- job-template:
+    name: test-job-template
+    triggers:
+    - gerrit:
+        projects:
+            !j2-yaml: |
+                {% for item in triggers %}
+                - branches:
+                  - branch-compare-type: PLAIN
+                    branch-pattern: '{{ item.branch }}'
+                  project-compare-type: REG_EXP
+                  project-pattern: '{{ item.repositories|join("|") }}'
+                {% endfor %}
+
+- project:
+    name: test-job-project
+
+    jobs:
+    - test-job-template:
+        triggers:
+        - repositories:
+            - a
+            - b
+            - c
+          branch: master
+        - repositories:
+            - d
+            - e
+            - f
+          branch: stable

tests/yamlparser/fixtures/jinja-yaml02.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild">
+      <autoRebuild>false</autoRebuild>
+      <rebuildDisabled>false</rebuildDisabled>
+    </com.sonyericsson.rebuild.RebuildSettings>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <com.sonyericsson.rebuild.RebuildSettings plugin="rebuild">
+      <autoRebuild>false</autoRebuild>
+      <rebuildDisabled>false</rebuildDisabled>
+    </com.sonyericsson.rebuild.RebuildSettings>
+    <jenkins.model.BuildDiscarderProperty>
+      <strategy class="hudson.tasks.LogRotator">
+        <daysToKeep>7</daysToKeep>
+        <numToKeep>-1</numToKeep>
+        <artifactDaysToKeep>-1</artifactDaysToKeep>
+        <artifactNumToKeep>-1</artifactNumToKeep>
+      </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-yaml02.yaml

@@ -0,0 +1,19 @@
+- job-template:
+    name: 'test-job-{variant}'
+    properties: !j2-yaml: |
+        - rebuild
+        {% if discard_old_builds|default %}
+        - build-discarder:
+            days-to-keep: 7
+        {% endif %}
+
+- project:
+    name: test-project
+
+    jobs:
+    - 'test-job-{variant}':
+        variant: abc
+
+    - 'test-job-{variant}':
+        variant: def
+        discard_old_builds: true

tests/yamlparser/fixtures/jinja-yaml03.groovy

@@ -0,0 +1,3 @@
+if (manager.logContains(".*no_jenkins.*")) {
+    manager.build.result = hudson.model.Result.NOT_BUILT
+}

tests/yamlparser/fixtures/jinja-yaml03.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers>
+    <org.jvnet.hudson.plugins.groovypostbuild.GroovyPostbuildRecorder>
+      <behavior>0</behavior>
+      <runForMatrixParent>false</runForMatrixParent>
+      <script>
+        <script>if (manager.logContains(&quot;.*no_jenkins.*&quot;)) {
+    manager.build.result = hudson.model.Result.NOT_BUILT
+}
+</script>
+        <sandbox>false</sandbox>
+      </script>
+    </org.jvnet.hudson.plugins.groovypostbuild.GroovyPostbuildRecorder>
+  </publishers>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-yaml03.yaml

@@ -0,0 +1,15 @@
+# the purpose of this test is to check if the piece of YAML generated by
+# !j2-yaml is deep-formatted properly; if not then double quotes introduced by
+# !include-raw-escape would be left untouched and passed down to the output XML
+# file, which would simply be wrong...
+
+- job-template:
+    name: 'test-job-template'
+    publishers: !j2-yaml: |
+        - groovy-postbuild:
+            script: !include-raw-escape: ./jinja-yaml03.groovy
+
+- project:
+    name: test-project
+    jobs:
+    - 'test-job-template'