Allow use of jinja templates in defaults

Jinja templates are not deep-copyable so they cannot be used in
"defautls" sections or to pick defualts for job groups or projects.

This works around the issue by waiting until we render the template to
construct the template itself.

Story: 2006431
Task: 36337
Change-Id: Ief31fdaac06bb14d0aaba71c8c0e658a7f861671

jenkins_jobs/local_yaml.py

@@ -545,15 +545,29 @@ class Jinja2Loader(CustomLoader):
     """A loader for Jinja2-templated files."""
 
     def __init__(self, contents, search_path):
-        self._template = jinja2.Template(contents)
-        self._template.environment.undefined = jinja2.StrictUndefined
-        self._template.environment.loader = jinja2.FileSystemLoader(search_path)
-        self._loader = self._template.environment.loader
+        # capture template contents and search paths on loader creation.
+        self._contents = contents
+        self._search_path = search_path
+        self._template = None
+        self._loader = None
+
+    def __deepcopy__(self, memo):
+        # Jinja 2 templates are not deepcopy-able so just pass around
+        # the search_path and contents.
+        return Jinja2Loader(self._contents, self._search_path)
 
     def format(self, **kwargs):
-        # For some reasons loader is overwritten with incorrect one during
-        # instance lifecycle. It's not very clear how to fix this properly,
-        # so we just overwrite with correct one
+        # Wait until first render call to create a template then save
+        # the template on this instance for faster rendering.
+        if not self._template:
+            self._template = jinja2.Template(self._contents)
+            self._template.environment.undefined = jinja2.StrictUndefined
+            self._template.environment.loader = jinja2.FileSystemLoader(
+                self._search_path
+            )
+            # Preserve this loader if it hasn't been overwritten
+            # elsewhere.
+            self._loader = self._template.environment.loader
         self._template.environment.loader = self._loader
         return self._template.render(kwargs)
 

tests/yamlparser/fixtures/jinja-string03.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>123</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/jinja-string03.yaml

@@ -0,0 +1,19 @@
+# Make sure Jinja subsittuions work from within "defaults"
+- defaults:
+    name: test-defaults
+    test_var:
+      !j2: "{% for x in [1, 2, 3] %}{{ x }}{% endfor %}"
+
+- project:
+    name: test-proj
+    jobs:
+      - test-jobs-{argument}:
+          argument:
+            - 1
+
+# This type of variable propagation only works in job templates.
+- job-template:
+    name: test-jobs-{argument}
+    defaults: test-defaults
+    builders:
+      - shell: "{test_var}"