diff --git a/jenkins_jobs/formatter.py b/jenkins_jobs/formatter.py index bd2576b9e..5bd1b7100 100644 --- a/jenkins_jobs/formatter.py +++ b/jenkins_jobs/formatter.py @@ -26,7 +26,7 @@ from jenkins_jobs.local_yaml import CustomLoader logger = logging.getLogger(__name__) -def deep_format(obj, paramdict, allow_empty=False): +def deep_format(obj, paramdict, allow_empty=False, template=True): """Apply the paramdict via str.format() to all string objects found within the supplied obj. Lists and dicts are traversed recursively.""" # YAML serialisation was originally used to achieve this, but that places @@ -50,13 +50,26 @@ def deep_format(obj, paramdict, allow_empty=False): elif isinstance(obj, list): ret = type(obj)() for item in obj: - ret.append(deep_format(item, paramdict, allow_empty)) + ret.append(deep_format(item, paramdict, + allow_empty=allow_empty, + template=template)) elif isinstance(obj, dict): ret = type(obj)() for item in obj: try: - ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \ - deep_format(obj[item], paramdict, allow_empty) + # deep_formatting dsl when not a job-template is not necessary + # as it will most likely result in keyerror due to trying + # to substitute values inside the dsl that do not exist. + if item not in ['dsl'] or template: + ret[CustomFormatter(allow_empty).format(item, + **paramdict)] = \ + deep_format(obj[item], paramdict, + allow_empty=allow_empty, + template=template) + else: + ret[CustomFormatter(allow_empty).format(item, + **paramdict)] = \ + obj[item] except KeyError as exc: missing_key = exc.args[0] desc = "%s parameter missing to format %s\nGiven:\n%s" % ( @@ -72,7 +85,8 @@ def deep_format(obj, paramdict, allow_empty=False): if isinstance(ret, CustomLoader): # If we have a CustomLoader here, we've lazily-loaded a template; # attempt to format it. - ret = deep_format(ret, paramdict, allow_empty=allow_empty) + ret = deep_format(ret, paramdict, allow_empty=allow_empty, + template=template) return ret diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index 213d8a7ab..75d1c5c65 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -491,6 +491,17 @@ class YamlIncludeJinja2(YamlIncludeRaw): return Jinja2Loader(contents, loader.search_path) +class YamlIncludeJinja2AsYaml(YamlIncludeJinja2): + yaml_tag = u'!include-jinja2-as-yaml:' + + @classmethod + def _from_file(cls, loader, node): + contents = cls._open_file(loader, node) + if isinstance(contents, LazyLoader): + return contents + return Jinja2LoaderAsYaml(contents, loader.search_path) + + class DeprecatedTag(BaseYAMLObject): @classmethod @@ -537,6 +548,14 @@ class Jinja2Loader(CustomLoader): return self._template.render(kwargs) +class Jinja2LoaderAsYaml(Jinja2Loader): + """A loader for Jinja2-templated files that renders yaml.""" + + def format(self, **kwargs): + raw_yaml = super(Jinja2LoaderAsYaml, self).format(**kwargs) + return yaml.load(raw_yaml) + + class CustomLoaderCollection(object): """Helper class to format a collection of CustomLoader objects""" def __init__(self, sequence): diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index 667ce426f..e8549c8f1 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -243,6 +243,16 @@ class YamlParser(object): if jobs_glob and not matches(job['name'], jobs_glob): logger.debug("Ignoring job {0}".format(job['name'])) continue + + # Attempt to format all parts of the job definition as they might + # be using custom loaders. + try: + job = deep_format(job, job, template=False) + except Exception: + logging.error( + "Failure formatting job '%s' with itself", job) + raise + logger.debug("Expanding job '{0}'".format(job['name'])) self._formatDescription(job) self.jobs.append(job) diff --git a/tests/yamlparser/fixtures/jinja-as-yaml-include01.xml b/tests/yamlparser/fixtures/jinja-as-yaml-include01.xml new file mode 100644 index 000000000..732bb2c4b --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-as-yaml-include01.xml @@ -0,0 +1,31 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + TEST_CHOICE + + + + a + b + c + + + + + + + + + + + diff --git a/tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml b/tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml new file mode 100644 index 000000000..2ea430f04 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml @@ -0,0 +1,7 @@ +- job: + name: test-job-as-yaml + parameters: + - choice: + name: TEST_CHOICE + choices: + !include-jinja2-as-yaml: jinja-as-yaml-include01.yaml.inc diff --git a/tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml.inc b/tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml.inc new file mode 100644 index 000000000..c1c3c8b3a --- /dev/null +++ b/tests/yamlparser/fixtures/jinja-as-yaml-include01.yaml.inc @@ -0,0 +1,3 @@ +{% for item in ['a', 'b', 'c'] %} +- {{ item }} +{% endfor -%}